diff --git a/README.md b/README.md index 7d77dab..7f46226 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,22 @@ cd rewards-farmer # Edit the included nouns.txt file to add or replace words as needed ``` +# Where search queries come from + +The bot needs short strings to type into Bing. Two backends produce them, set with `QUERY_SOURCE`: + +| `QUERY_SOURCE` | Needs | Notes | +| --- | --- | --- | +| `llm` (default) | Ollama account + model | Current behaviour, unchanged | +| `trends` | nothing | Google Trends, Wikipedia and Bing autosuggest | + +```sh +QUERY_SOURCE=trends python src/main.py # bash +$env:QUERY_SOURCE="trends"; python src/main.py # PowerShell +``` + +`trends` needs no account, no API key and no model download, so the Ollama setup below is optional if you use it. If every feed is unreachable it falls back to `nouns.txt` rather than failing the run. + You should also have an Ollama account created (for the LLM), the `ollama` tool installed, and you should have signed in to the Ollama CLI via the command line using `ollama signin`. This project will use a minimal amount of Ollama cloud usage using `gemma4:cloud`. If you wish to use a different model, please change the `model` parameter in the `get_ollama_response` function in `src/llm_utils.py`. You must also provide an image for the script to upload to complete the visual search task. Currently, this image is named `keypress_times.png` and is located in the root directory of the project (yes, I used a random image from my keyboard analysis to do this). You may provide an image of your own, just ensure that the absolute path of the image is placed in the `VISUAL_SEARCH_IMAGE_PATH` constant at the top of `rewards_tasks.py`. diff --git a/src/queries.py b/src/queries.py new file mode 100644 index 0000000..269d1d8 --- /dev/null +++ b/src/queries.py @@ -0,0 +1,64 @@ +"""Where search queries come from. + +Two backends. `llm` is the default and is unchanged, so nothing about an +existing setup moves. `trends` uses public feeds and needs no account, no +model and no key, which is the difference between running this in five +minutes and installing Ollama first. + + QUERY_SOURCE=trends python src/main.py + +The LLM's whole job in this project is producing short strings to type into +Bing, and Bing's own autosuggest answers that question directly. +""" + +import os + +import llm_utils +import query_sources + +LLM = "llm" +TRENDS = "trends" + +DEFAULT_SOURCE = LLM + +ENV_VAR = "QUERY_SOURCE" + + +def selected_source() -> str: + """Read on each call so a test can change it without reimporting.""" + choice = os.environ.get(ENV_VAR, DEFAULT_SOURCE).strip().lower() + + return choice if choice in (LLM, TRENDS) else DEFAULT_SOURCE + + +def search_query_for_task(task_description: str) -> str: + """A query for one "Search on Bing for X" card.""" + if selected_source() == TRENDS: + query = query_sources.query_from_task_description(task_description) + + if query: + return query + + # Every feed was unreachable. The description still contains the topic, + # so a trimmed version beats skipping the card entirely. + print("[WARNING] No query source reachable, using the task description as written.") + + return task_description.lower() + + return llm_utils.get_search_query_from_task_description(task_description) + + +def related_queries(count: int): + """`count` queries for the daily search quota.""" + if selected_source() == TRENDS: + queries = query_sources.related_queries(count) + + if queries: + return queries + + print("[WARNING] No query source reachable, falling back to the wordlist.") + + # nouns.txt is already in the repo for exactly this kind of seed. + return [llm_utils.get_random_noun() for _ in range(count)] + + return llm_utils.get_related_search_queries(llm_utils.get_random_noun(), num_queries=count) diff --git a/src/query_sources.py b/src/query_sources.py new file mode 100644 index 0000000..b9bdee4 --- /dev/null +++ b/src/query_sources.py @@ -0,0 +1,198 @@ +"""Search queries from public feeds instead of a language model. + +The LLM in this project has one job: produce short strings to type into Bing. +That is worth an Ollama account and a model download if you want it, but it is +not the only way to get a search query, and it is the piece that stops someone +running the bot in five minutes. + +Three keyless sources, all stdlib, no new dependencies: + + Google Trends RSS real queries people are typing right now + Wikipedia most-read topic seeds, useful when trends is unavailable + Bing autosuggest expands a seed into related queries + +Autosuggest is what makes the chaining work. Asking Bing what follows a term +gives queries Bing itself expects, which is closer to what the LLM prompt was +reaching for than a model guessing in the dark. + +Every source degrades rather than raises. A search that does not happen costs +points; a run that dies costs the rest of the day's points too. +""" + +import json +import random +import re +import urllib.error +import urllib.parse +import urllib.request +from datetime import date, timedelta + +TRENDS_URL = "https://trends.google.com/trending/rss?geo={geo}" +WIKIPEDIA_URL = "https://en.wikipedia.org/api/rest_v1/feed/featured/{y}/{m:02d}/{d:02d}" +AUTOSUGGEST_URL = "https://api.bing.com/osjson.aspx?query={query}" + +# A browser agent: trends and the Wikipedia REST feed both answer differently +# to an unfamiliar client, and one of them refuses outright. +USER_AGENT = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36" +) + +REQUEST_TIMEOUT = 15 + +# Words that make a query read as an instruction rather than a search. The +# task descriptions are phrased at the user, "Search on Bing to compare +# checking accounts", and typing that verbatim searches for the sentence. +INSTRUCTION_WORDS = { + "search", "searching", "bing", "on", "to", "the", "a", "an", "for", "your", + "you", "use", "using", "find", "get", "with", "and", "or", "of", "in", + "at", "by", "now", "today", "this", "that", "these", "those", "learn", + "discover", "explore", "check", "see", "our", "more", "about", "how", +} + + +def _fetch(url: str) -> str | None: + """Body of a GET, or None. Never raises: callers fall through to the next source.""" + request = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) + + try: + with urllib.request.urlopen(request, timeout=REQUEST_TIMEOUT) as response: + return response.read().decode("utf-8", "replace") + except (urllib.error.URLError, urllib.error.HTTPError, OSError, ValueError): + return None + + +def trending_queries(geo: str = "US") -> list[str]: + """Queries currently trending on Google, most popular first. + + These are real searches rather than descriptions of searches, which is + exactly the shape wanted here. + """ + body = _fetch(TRENDS_URL.format(geo=urllib.parse.quote(geo))) + + if not body: + return [] + + # The channel carries a of its own before any item, so the first + # match is the feed name rather than a query. + titles = re.findall(r"<title>(?:<!\[CDATA\[)?(.*?)(?:\]\]>)?", body, re.S) + + return [_clean(t) for t in titles[1:] if _clean(t)] + + +def wikipedia_topics(days_ago: int = 1) -> list[str]: + """Most-read Wikipedia articles, as topic seeds. + + Yesterday by default: today's feed is not published until the day is over. + """ + day = date.today() - timedelta(days=days_ago) + body = _fetch(WIKIPEDIA_URL.format(y=day.year, m=day.month, d=day.day)) + + if not body: + return [] + + try: + payload = json.loads(body) + except json.JSONDecodeError: + return [] + + articles = payload.get("mostread", {}).get("articles", []) + titles = [a.get("titles", {}).get("normalized", "") for a in articles] + + # Wikipedia's own chrome outranks real topics most days. + skipped = ("Main Page", "Special:", "Wikipedia:", "Portal:") + + return [ + _clean(t) for t in titles + if t and not t.startswith(skipped) and _clean(t) + ] + + +def suggestions(seed: str) -> list[str]: + """What Bing suggests for a term, which is what Bing expects to be asked.""" + if not seed.strip(): + return [] + + body = _fetch(AUTOSUGGEST_URL.format(query=urllib.parse.quote(seed))) + + if not body: + return [] + + try: + payload = json.loads(body) + except json.JSONDecodeError: + return [] + + # Opensearch shape: [term, [suggestions], ...] + if not isinstance(payload, list) or len(payload) < 2 or not isinstance(payload[1], list): + return [] + + return [_clean(s) for s in payload[1] if _clean(s)] + + +def _clean(text: str) -> str: + """Strip markup, collapse whitespace and drop punctuation Bing does not need.""" + text = re.sub(r"<[^>]+>", " ", text or "") + text = re.sub(r"[\"'?!,;:]", " ", text) + + return " ".join(text.split()).strip().lower() + + +def query_from_task_description(description: str) -> str | None: + """A search query for a task phrased as an instruction. + + "Search on Bing to compare checking and savings account options" becomes + the content words, then whatever Bing suggests for them, so the query is + one Bing already recognises rather than the sentence itself. + """ + words = [w for w in _clean(description).split() if w not in INSTRUCTION_WORDS] + + if not words: + return None + + seed = " ".join(words[:6]) + options = suggestions(seed) + + # Prefer a suggestion, since it is a query Bing has seen. The trimmed + # sentence is a reasonable fallback and still beats typing the imperative. + return options[0] if options else seed + + +def related_queries(count: int, seed: str | None = None) -> list[str]: + """`count` distinct queries, branching out the way the LLM prompt asks for. + + Trending queries first, since they need no expansion at all, then Bing's + suggestions for each to reach the requested number. + """ + collected: list[str] = [] + seen: set[str] = set() + + def take(candidates): + for candidate in candidates: + if candidate and candidate not in seen and len(candidate) > 2: + seen.add(candidate) + collected.append(candidate) + + if len(collected) >= count: + return True + return False + + if seed: + take(suggestions(seed)) + + if len(collected) < count and take(trending_queries()): + return collected[:count] + + if len(collected) < count: + take(wikipedia_topics()) + + # Expand what we have until the count is met. Iterating over a snapshot + # because take() appends to the same list. + for term in list(collected): + if len(collected) >= count: + break + + take(suggestions(term)) + + # Nothing reachable: let the caller decide, rather than typing junk. + return collected[:count] diff --git a/src/rewards_tasks.py b/src/rewards_tasks.py index 55773db..b015b0e 100644 --- a/src/rewards_tasks.py +++ b/src/rewards_tasks.py @@ -10,7 +10,7 @@ from selenium.webdriver.remote.webelement import WebElement from selenium.webdriver.common.action_chains import ActionChains from selenium.common.exceptions import StaleElementReferenceException, TimeoutException, NoSuchElementException import tab_utils -import llm_utils +import queries import mouse_trajectory import mimic_typing import element_selectors @@ -93,7 +93,7 @@ class RewardsTaskUtils: for card in explore_on_bing_links: desc = self.elements.extract_card_descriptions(card) - query = llm_utils.get_search_query_from_task_description(desc) + query = queries.search_query_for_task(desc) self.move_to_and_click(card) self.tab_utils.switch_to_other_tab() @@ -220,9 +220,7 @@ class RewardsTaskUtils: # search bar should be auto-focused for i, query in enumerate( - llm_utils.get_related_search_queries( - llm_utils.get_random_noun(), num_queries=count - ) + queries.related_queries(count) ): self.keyboard.send_keys(f"{query} -noai{Keys.ENTER}")