diff --git a/docs/api.rst b/docs/api.rst new file mode 100644 index 0000000..75e7310 --- /dev/null +++ b/docs/api.rst @@ -0,0 +1,24 @@ +API reference +============= + +.. automodule:: wabot + :members: browser, sessions, destroy + +.. autoclass:: wabot.Browser + :members: + +.. autoclass:: wabot.Page + :members: + +.. automodule:: wabot.fields + :members: PageObject, TextField, SelectField, CheckField, NullField + +.. automodule:: wabot.pacing + :members: + +.. automodule:: wabot.sessions + :members: + :no-index: + +.. automodule:: wabot.hosts + :members: ExternalServer, ManagedService, attach, service_alive diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..e51236a --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,14 @@ +project = "wabot" +author = "Mathew Guest" +release = "0.2.0" + +extensions = [ + "sphinx.ext.autodoc", + "sphinx.ext.napoleon", +] + +# sphinx_rtd_theme registers itself via entry point — do NOT add to extensions +html_theme = "sphinx_rtd_theme" + +autodoc_member_order = "bysource" +napoleon_google_docstring = True diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..5c3462c --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,13 @@ +wabot +===== + +Stateful Selenium browser automation with sessions that survive the +Python process. + +.. toctree:: + :maxdepth: 2 + + quickstart + sessions + page-objects + api diff --git a/docs/page-objects.rst b/docs/page-objects.rst new file mode 100644 index 0000000..ee269d6 --- /dev/null +++ b/docs/page-objects.rst @@ -0,0 +1,47 @@ +Page objects +============ + +Model each website page as a :class:`wabot.Page` subclass with a +class-level ``elements`` map; drive them through one +:class:`wabot.Browser`. + +.. code-block:: python + + class Inventory(wabot.Page): + elements = { + "search": ("el", ("id", "search-box")), + "region": ("select", ("id", "region")), + "instock": ("checkbox", ("id", "in-stock-only")), + "rows": ("els", ("css selector", "table#items tr")), + } + +Element types +------------- + +============ ======================================= ===================== +Type key Returned wrapper Main methods +============ ======================================= ===================== +``el`` :class:`wabot.TextField` get_value / set_value +``select`` :class:`wabot.SelectField` set_value(value|text) +``checkbox`` :class:`wabot.CheckField` get/set_checked +``els`` list of raw WebElements (selenium API) +a class your :class:`wabot.PageObject` subclass whatever you define +============ ======================================= ===================== + +Missing elements come back as a falsy :class:`wabot.NullField`, so guard +with ``if el:`` instead of try/except. + +Inheritance +----------- + +Element lookup walks the MRO: subclasses inherit the parent map and can +override individual keys — useful for sites with shared page chrome. + +Verification and delegation +--------------------------- + +``Browser.set_page(PageClass)`` refuses the switch if the page's +``verify()`` returns False. Once set, unknown attributes on the Browser +delegate to the current page: ``bot.log_in(...)`` calls +``bot.page.log_in(...)``. After any trapped failure the Browser refuses +further actions until ``bot.reset()``. diff --git a/docs/quickstart.rst b/docs/quickstart.rst new file mode 100644 index 0000000..a0e1f36 --- /dev/null +++ b/docs/quickstart.rst @@ -0,0 +1,44 @@ +Quickstart +========== + +Install +------- + +.. code-block:: bash + + uv add wabot # or: pip install . + +You also need Firefox or Chromium. Driver binaries (geckodriver / +chromedriver) are found on PATH or downloaded automatically by Selenium +Manager. + +First scrape +------------ + +.. code-block:: python + + import wabot + + class Login(wabot.Page): + elements = { + "username": ("el", ("id", "username")), + "password": ("el", ("id", "password")), + "submit": ("el", ("id", "submit-btn")), + } + + def verify(self): + return "Login" in self.driver.title + + def log_in(self, user, password): + self["username"].set_value(user) + self["password"].set_value(password) + return self["submit"].click_and_go() + + bot = wabot.browser(browser="firefox", headless=True) + bot.driver.get("https://example.com/login") + bot.set_page(Login) + bot.log_in("me", "secret") + +By default wabot paces itself like a human (multi-second random delays, +randomized click positions). Pass ``pacing=wabot.NoPacing()`` for full +speed on sites you trust. diff --git a/docs/sessions.rst b/docs/sessions.rst new file mode 100644 index 0000000..99590fc --- /dev/null +++ b/docs/sessions.rst @@ -0,0 +1,51 @@ +Persistent sessions +=================== + +The flagship feature: a browser created with a ``session`` name survives +the Python process, and any later process can pick it up by name. + +.. code-block:: python + + import wabot + + bot = wabot.browser(session="scraper1", browser="firefox") + bot.driver.get("https://example.com/login") + # ... log in, then the process exits; the browser stays open ... + + # hours later, a different process: + bot = wabot.browser(session="scraper1", browser="firefox") + print(bot.driver.current_url) # still logged in + +How it works +------------ + +A session is just ``(executor_url, session_id)`` stored as JSON in the +platform data dir. Nothing is pickled. The browser is hosted by a driver +service that outlives Python: + +* **Managed (default):** wabot spawns chromedriver/geckodriver as a + detached process and records its port and pid. +* **External:** pass ``host="http://localhost:4444"`` to use a Selenium + Grid or standalone server you run yourself. wabot then never kills the + server — only the session. + +Reattaching constructs a driver that *adopts* the saved session id +instead of creating a new session. Dead sessions (browser closed, server +gone, record older than 3 days) are pruned automatically and a fresh +browser is created instead. + +Housekeeping +------------ + +.. code-block:: python + + wabot.sessions() # ["scraper1", ...] + wabot.destroy("scraper1") # quit browser, stop managed service, forget + +Selenium Grid caveat +-------------------- + +Grid 4 kills idle sessions after **5 minutes** by default. If your +process sleeps longer between reattaches, raise it:: + + java -jar selenium-server.jar standalone --session-timeout 86400