Switch to using web-based activation (#137)

This updates `user add` to use the same web-based activation approach
used by Kobo e-readers, rather than trying to scrape the login page.

This has three main benefits:

 1. No need for the use to provide their password to kobodl. Instead,
    they log in using the Kobo website from their browser.
 2. Avoids the step of using devtools to extract a captcha code.
 3. (Hopefully) less prone to breakage, since it doesn't rely on
    scraping the login page.

This change was adapted from TnS-hun/kobo-book-downloader commit
04d8c42.

Co-authored-by: Brandon Davis <git@subdavis.com>

Author: rkjnsn

README.md

@@ -199,26 +199,8 @@ kobodl --version
 kobodl --debug [OPTIONS] COMMAND [ARGS]...
 ```
 
-## Getting a reCAPTCHA code
-
-Adding a user requires a bit of hackery to get a reCAPTCHA code from Kobo's website. This GIF helps to explain how to do that.
-
-![Gif explaining how to get reCAPTHCA](docs/captcha.gif)
-
 ## Troubleshooting
 
-> I can't log in.  My credentials are rejected.
-
-You must have a kobo email login for this tool to work (you can't use an external provider like Google or Facebook). However, you can create a NEW kobo account and link it with your existing account on the user profile page. Go to `My Account -> Account Settings` to link your new kobo login.
-
-> I can't log in.  I get a message saying "The page format might have changed"
-
-This happens from time to time, maybe once or twice a year.  Kobo changes their login page and makes it hard for the tool to parse out the necessary information.  Please open an issue.
-
-> I can't log in, there's a problem with reading the captcha
-
-The clipboard interaction doesn't work for everyone.  Try supplying the captcha using `kobodl user add --captcha "YOUR_CAPTCHA_CODE"`.
-
 > Some of my books are missing!
 
 Try `kobodl book list --read` to show all "finished" and "archived" books.  You can manage your book status on [the library page](https://kobo.com/library).  Try changing the status using the "..." button.
@@ -256,7 +238,7 @@ poetry run tox -e type
 
 ## Notes
 
-kobo-book-downloader will prompt for your [Kobo](https://www.kobo.com/) e-mail address and password. Once it has successfully logged in, it won't ask for them again. Your password will not be stored on disk; Kobodl uses access tokens after the initial login.
+kobo-book-downloader uses the same web-based activation method to login as the Kobo e-readers. You will have to open an activation link—that uses the official [Kobo](https://www.kobo.com/) site—in your browser and enter the code. You might need to login if kobo.com asks you to. Once kobo-book-downloader has successfully logged in, it won't ask for the activation again. kobo-book-downloader doesn't store your Kobo password in any form; it works with access tokens.
 
 Credit recursively to [kobo-book-downloader](https://github.com/TnS-hun/kobo-book-downloader) and the projects that lead to it.
 

kobodl/actions.py

@@ -161,12 +161,12 @@ def ListBooks(users: List[User], listAll: bool, exportFile: Union[TextIO, None])
             )
 
 
-def Login(user: User, password: str, captcha: str) -> None:
+def Login(user: User) -> None:
     '''perform device initialization and get token'''
     kobo = Kobo(user)
     kobo.AuthenticateDevice()
     kobo.LoadInitializationSettings()
-    kobo.Login(user.Email, password, captcha)
+    kobo.Login()
 
 
 def GetBookOrBooks(

kobodl/commands/user.py

@@ -44,40 +44,10 @@ def list(ctx, identifier):
 
 
 @user.command(name='add', help='add new user')
-@click.option('--email', prompt=True, hide_input=False, type=click.STRING, help="kobo.com email.")
-@click.option('--captcha', type=click.STRING, help="kobo.com captcha.")
-@click.password_option(help="kobo.com password (not stored)")
 @click.pass_obj
-def add(ctx, email, captcha, password):
-    user = User(Email=email)
-    click.echo(
-        """
-    1. Open https://authorize.kobo.com/signin in a private/incognito window in your browser.
-    2. wait till the page loads (do not login!)
-    3. open the developer tools (use F12 in Firefox/Chrome, or right-click and choose "inspect")
-    4. select the console tab,
-    5. copy-paste the following code to the console there and then press Enter.
-
-    var newCaptchaDiv = document.createElement("div");
-    newCaptchaDiv.id = "new-hcaptcha-container";
-    var siteKey = document.getElementById('hcaptcha-container').getAttribute('data-sitekey');
-    document.body.replaceChildren(newCaptchaDiv);
-    grecaptcha.render(newCaptchaDiv.id, {
-        sitekey: siteKey,
-        callback: function(r) {console.log("Captcha response:");console.log(r);}
-    });
-    console.log('Click the checkbox to get the code');
-
-    A captcha should show up below the Sign-in form. Once you solve the captcha its response will be written
-    below the pasted code in the browser's console. Copy the response (the line below "Captcha response:")
-    and paste it here.  It will be very long!
-    """
-    )
-    if not captcha:
-        input('Press enter after copying the captcha code...')
-        captcha = pyperclip.paste().strip()
-        click.echo(f'Read captcha code from clipboard: {captcha}')
-    actions.Login(user, password, captcha)
+def add(ctx):
+    user = User()
+    actions.Login(user)
     Globals.Settings.UserList.users.append(user)
     Globals.Settings.Save()
     click.echo('Login Success. Try to list your books with `kobodl book list`')

kobodl/kobo.py

@@ -3,9 +3,11 @@
 import html
 import os
 import re
+import secrets
+import string
 import sys
+import time
 import urllib
-import uuid
 from enum import Enum
 from shutil import copyfile
 from typing import Dict, Tuple
@@ -47,10 +49,14 @@ class KoboException(Exception):
 
 class Kobo:
     Affiliate = "Kobo"
-    ApplicationVersion = "8.11.24971"
-    DefaultPlatformId = "00000000-0000-0000-0000-000000004000"
+    ApplicationVersion = "4.38.23171"
+    DefaultPlatformId = "00000000-0000-0000-0000-000000000373"
     DisplayProfile = "Android"
-    UserAgent = "Mozilla/5.0 (Linux; Android 6.0; Google Nexus 7 2013 Build/MRA58K; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/74.0.3729.186 Safari/537.36 KoboApp/8.40.2.29861 KoboPlatform Id/00000000-0000-0000-0000-000000004000 KoboAffiliate/Kobo KoboBuildFlavor/global"
+    DeviceModel = "Kobo Aura ONE"
+    DeviceOs = "3.0.35+"
+    DeviceOsVersion = "NA"
+    # Use the user agent of the Kobo e-readers
+    UserAgent = "Mozilla/5.0 (Linux; U; Android 2.0; en-us;) AppleWebKit/538.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/538.1 (Kobo Touch 0373/4.38.23171)"
 
     def __init__(self, user: User):
         self.InitializationSettings = {}
@@ -67,6 +73,50 @@ def __GetHeaderWithAccessToken(self) -> dict:
         headers = {"Authorization": authorization}
         return headers
 
+    def __WaitTillActivation(self, activationCheckUrl) -> Tuple[str, str]:
+        while True:
+            print("Waiting for you to finish the activation...")
+            time.sleep(5)
+
+            response = self.Session.get(activationCheckUrl)
+            response.raise_for_status()
+            jsonResponse = response.json()
+            if jsonResponse["Status"] == "Complete":
+                return jsonResponse["UserEmail"], jsonResponse["UserId"], jsonResponse["UserKey"]
+
+    def __ActivateOnWeb(self) -> Tuple[str, str]:
+        print( "Initiating web-based activation" )
+
+        params = {
+            "pwspid": Kobo.DefaultPlatformId,
+            "wsa": Kobo.Affiliate,
+            "pwsdid": self.user.DeviceId,
+            "pwsav": Kobo.ApplicationVersion,
+            "pwsdm": Kobo.DefaultPlatformId, # In the Android app this is the device model but Nickel sends the platform ID...
+            "pwspos": Kobo.DeviceOs,
+            "pwspov": Kobo.DeviceOsVersion,
+        }
+
+        response = self.Session.get("https://auth.kobobooks.com/ActivateOnWeb", params=params)
+        response.raise_for_status()
+        htmlResponse = response.text
+
+        match = re.search('data-poll-endpoint="([^"]+)"', htmlResponse)
+        if match is None:
+            raise KoboException(
+                "Can't find the activation poll endpoint in the response. The page format might have changed."
+            )
+        activationCheckUrl = "https://auth.kobobooks.com" + html.unescape(match.group(1))
+
+        match = re.search(r"""qrcodegenerator/generate.+?%26code%3D(\d+)'""", htmlResponse)
+        if match is None:
+            raise KoboException(
+                "Can't find the activation code in the response. The page format might have changed."
+            )
+        activationCode = match.group(1)
+
+        return activationCheckUrl, activationCode
+
     def __RefreshAuthentication(self) -> None:
         headers = self.__GetHeaderWithAccessToken()
 
@@ -131,43 +181,6 @@ def ReauthenticationHook(r, *args, **kwargs):
 
         return {"response": ReauthenticationHook}
 
-    def __GetExtraLoginParameters(self) -> Tuple[str, str, str]:
-        signInUrl = self.InitializationSettings["sign_in_page"]
-
-        params = {
-            "wsa": Kobo.Affiliate,
-            "pwsav": Kobo.ApplicationVersion,
-            "pwspid": Kobo.DefaultPlatformId,
-            "pwsdid": self.user.DeviceId,
-        }
-
-        response = self.Session.get(signInUrl, params=params)
-        response.raise_for_status()
-        htmlResponse = response.text
-
-        # The link can be found in the response ('<a class="kobo-link partner-option kobo"') but this will do for now.
-        parsed = urllib.parse.urlparse(signInUrl)
-        koboSignInUrl = parsed._replace(query=None, path="/ww/en/signin/signin").geturl()
-
-        match = re.search(r"""\?workflowId=([^"]{36})""", htmlResponse)
-        if match is None:
-            raise KoboException(
-                "Can't find the workflow ID in the login form. The page format might have changed."
-            )
-        workflowId = html.unescape(match.group(1))
-
-        match = re.search(
-            r"""<input name="__RequestVerificationToken" type="hidden" value="([^"]+)" />""",
-            htmlResponse,
-        )
-        if match is None:
-            raise KoboException(
-                "Can't find the request verification token in the login form. The page format might have changed."
-            )
-        requestVerificationToken = html.unescape(match.group(1))
-
-        return koboSignInUrl, workflowId, requestVerificationToken
-
     def __GetMyBookListPage(self, syncToken: str) -> Tuple[list, str]:
         url = self.InitializationSettings["library_sync"]
         headers = self.__GetHeaderWithAccessToken()
@@ -286,6 +299,11 @@ def __DownloadAudiobook(self, url, outputPath: str) -> None:
                 for chunk in response.iter_content(chunk_size=1024 * 256):
                     f.write(chunk)
 
+    @staticmethod
+    def __GenerateRandomHexDigitString( length: int ) -> str:
+        id = "".join( secrets.choice( string.hexdigits ) for _ in range( length ) )
+        return id.lower()
+
     # PUBLIC METHODS:
     @staticmethod
     def GetProductId(bookMetadata: dict) -> str:
@@ -297,7 +315,8 @@ def GetProductId(bookMetadata: dict) -> str:
     # user key can't be used for anything.
     def AuthenticateDevice(self, userKey: str = "") -> None:
         if len(self.user.DeviceId) == 0:
-            self.user.DeviceId = str(uuid.uuid4())
+            self.user.DeviceId = Kobo.__GenerateRandomHexDigitString(64)
+            self.user.SerialNumber = Kobo.__GenerateRandomHexDigitString(32)
             self.user.AccessToken = ""
             self.user.RefreshToken = ""
 
@@ -307,6 +326,7 @@ def AuthenticateDevice(self, userKey: str = "") -> None:
             "ClientKey": base64.b64encode(Kobo.DefaultPlatformId.encode()).decode(),
             "DeviceId": self.user.DeviceId,
             "PlatformId": Kobo.DefaultPlatformId,
+            "SerialNumber": self.user.SerialNumber,
         }
 
         if len(userKey) > 0:
@@ -447,53 +467,23 @@ def LoadInitializationSettings(self) -> None:
             print(response.reason, response.text)
             raise err
 
-    def Login(self, email: str, password: str, captcha: str) -> None:
-        (
-            signInUrl,
-            workflowId,
-            requestVerificationToken,
-        ) = self.__GetExtraLoginParameters()
-
-        postData = {
-            "LogInModel.WorkflowId": workflowId,
-            "LogInModel.Provider": Kobo.Affiliate,
-            "ReturnUrl": "",
-            "__RequestVerificationToken": requestVerificationToken,
-            "LogInModel.UserName": email,
-            "LogInModel.Password": password,
-            "g-recaptcha-response": captcha,
-            "h-captcha-response": captcha,
-        }
-
-        response = self.Session.post(signInUrl, data=postData)
-        debug_data("Login", response.text)
-        response.raise_for_status()
-        htmlResponse = response.text
-
-        match = re.search(r"'(kobo://UserAuthenticated\?[^']+)';", htmlResponse)
-        if match is None:
-            soup = BeautifulSoup(htmlResponse, 'html.parser')
-            errors = soup.find(class_='validation-summary-errors') or soup.find(
-                class_='field-validation-error'
-            )
-            if errors:
-                raise KoboException('Login Failure! ' + errors.text)
-            else:
-                with open('loginpage_error.html', 'w') as loginpagefile:
-                    loginpagefile.write(htmlResponse)
-                raise KoboException(
-                    "Authenticated user URL can't be found. The page format might have changed!\n\n"
-                    "The bad page has been written to file 'loginpage_error.html'.  \n"
-                    "You should open an issue on GitHub and attach this file for help: https://github.com/subdavis/kobo-book-downloader/issues\n"
-                    "Please be sure to remove any personally identifying information from the file."
-                )
-
-        url = match.group(1)
-        parsed = urllib.parse.urlparse(url)
-        parsedQueries = urllib.parse.parse_qs(parsed.query)
-        self.user.UserId = parsedQueries["userId"][
-            0
-        ]  # We don't call self.Settings.Save here, AuthenticateDevice will do that if it succeeds.
-        userKey = parsedQueries["userKey"][0]
-
+    def Login( self ) -> None:
+        activationCheckUrl, activationCode = self.__ActivateOnWeb()
+
+        print("")
+        print("kobo-book-downloader uses the same web-based activation method to log in as the")
+        print("Kobo e-readers. You will have to open the link below in your browser and enter")
+        print("the code. You might need to login if kobo.com asks you to.")
+        print("")
+        print(f"Open https://www.kobo.com/activate and enter {activationCode}.")
+        print("")
+        print("kobo-book-downloader will wait now and periodically check for the activation to complete.")
+        print("")
+
+        userEmail, userId, userKey = self.__WaitTillActivation( activationCheckUrl )
+        print("")
+
+        # We don't call Settings.Save here, AuthenticateDevice will do that if it succeeds.
+        self.user.Email = userEmail
+        self.user.UserId = userId
         self.AuthenticateDevice(userKey)

kobodl/settings.py

@@ -8,8 +8,9 @@
 @dataclass_json
 @dataclasses.dataclass
 class User:
-    Email: str
+    Email: str = ""
     DeviceId: str = ""
+    SerialNumber: str = ""
     AccessToken: str = ""
     RefreshToken: str = ""
     UserId: str = ""