Merge branch 'master' of https://github.com/pgalko/athlete_data_warehouse

Author: pgalko

docker/athlete_data_warehouse_docker.zip

@@ -15,6 +15,19 @@ RUN pip install --no-cache-dir -r requirements.txt
 # Install pg_dump
 RUN apt-get update && apt-get -y install postgresql-client
 
+# Install Chrome and chromedriver
+# Install wget
+RUN apt-get install -y wget
+# Download and Install chrome
+RUN wget -q https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb
+RUN apt-get install -y ./google-chrome-stable_current_amd64.deb
+# Download and Install chromedriver. The version here depends on what what version of chrome gets installed above !
+RUN wget https://chromedriver.storage.googleapis.com/104.0.5112.79/chromedriver_linux64.zip
+RUN unzip chromedriver_linux64.zip
+RUN mv chromedriver /usr/bin/chromedriver
+RUN chown root:root /usr/bin/chromedriver
+RUN chmod +x /usr/bin/chromedriver
+
 # define the port number the container should expose
 EXPOSE 5000
 

src/Dockerfile

@@ -145,9 +145,35 @@ def auto_synch(ath_un, db_name, db_host, superuser_un, superuser_pw, gc_username
             finally:
                 if conn is not None:
                     conn.close()
-        #Periodic daily auto_synch(2days)
-        else:     
-            start_date = datetime.datetime.today() - datetime.timedelta(days=2) #PG: start_day = one day before yesterday 
+        #Periodic daily auto_synch(2 days back from last_synch)
+        #Retrieve last_synch from db_info, subtract 2 days and use as a start date for the autosynch
+        else:   
+            sql_select_last_synch ="""     
+            SELECT last_synch FROM db_info WHERE db_name= %s;
+            """
+            try:
+                params = config(filename="encrypted_settings.ini", section="postgresql", encr_pass=encr_pass)
+                postgres_db = params.get("database")
+                postgres_un = params.get("user")
+                postgres_pw = params.get("password")
+                postgres_host = params.get("host") 
+
+                conn_localhost = psycopg2.connect(host=postgres_host, dbname=postgres_db, user=postgres_un, password=postgres_pw)
+                cur_localhost = conn_localhost.cursor() 
+                cur_localhost.execute(sql_select_last_synch,(db_name,))
+                conn_localhost.commit()
+                result = cur_localhost.fetchone()
+                last_synch_str = result[0]
+                last_synch_dt = datetime.datetime.strptime(last_synch_str,"%Y-%m-%d %H:%M:%S")
+                start_date = last_synch_dt - datetime.timedelta(days=2) #PG: start_day = 2 days before last_synch
+            except (Exception, psycopg2.DatabaseError) as error:
+                with ErrorStdoutRedirection(ath_un):
+                    print((str(datetime.datetime.now()) + ' [' + sys._getframe().f_code.co_name + ']' + ' Error on line {}'.format(sys.exc_info()[-1].tb_lineno) + '  ' + str(error)))
+            finally:
+                if conn_localhost is not None:
+                    conn_localhost.close
+    
+            #start_date = datetime.datetime.today() - datetime.timedelta(days=2) #PG: start_day = one day before yesterday 
         #Calculate end_date
         end_date = datetime.datetime.today() - datetime.timedelta(days=1) #PG: end_date = yesterday
         end_date_today = datetime.datetime.today() #"PG: end_date_today = today

src/main_data_autosynch.py

@@ -58,8 +58,69 @@ def dwnld_insert_nutrition(mfp_username,mfp_password,ath_un,start_date,end_date,
         print("Attempting to login to MFP...")     
     with ProgressStdoutRedirection(ath_un):
         print("Attempting to login to MFP...")
+
+    #----------- 2022/08/29 --------
+    # The below code is to incorporate Selenium to retrieve the login cookies, instantiate the cookiejar and pass to myfitnesspall.Client().
+    # It is a temporary workaround to address MFP introducing hidden captcha on 25th of Aug.
+    # For this to work you will need to download the chromedriver that matches the version of yur chrome browser, and place it in /work_dir/temp/.
+    # https://chromedriver.chromium.org/getting-started
+    from selenium import webdriver
+    from selenium.webdriver.chrome.options import Options
+    from http.cookiejar import Cookie,CookieJar
+    import time
+
+    def to_cookielib_cookie(selenium_cookie):
+        if 'expiry' in selenium_cookie:
+            expires=selenium_cookie['expiry']
+        else:
+            expires=None
+        return Cookie(
+            version=0,
+            name=selenium_cookie['name'],
+            value=selenium_cookie['value'],
+            port='80',
+            port_specified=False,
+            domain=selenium_cookie['domain'],
+            domain_specified=True,
+            domain_initial_dot=False,
+            path=selenium_cookie['path'],
+            path_specified=True,
+            secure=selenium_cookie['secure'],
+            expires=expires,
+            discard=False,
+            comment=None,
+            comment_url=None,
+            rest=None,
+            rfc2109=False
+        )
+
+    def put_cookies_in_jar(selenium_cookies, cookie_jar):
+        for cookie in selenium_cookies:
+            cookie_jar.set_cookie(to_cookielib_cookie(cookie))
+
+    cj = CookieJar()
+    options = Options()
+    options.add_argument('--no-sandbox')
+    options.add_argument('--headless')
+    options.add_argument('--disable-dev-shm-usage')
+    options.add_argument("user-data-dir={}".format(PID_FILE_DIR+'selenium'))
+    driver = webdriver.Chrome(PID_FILE_DIR+'chromedriver.exe',options=options)# Modify this line if your chromedrive executable is located at a different path. The dafault location is /work_dir/temp.
+    driver.get('https://www.myfitnesspal.com/account/login') 
+    email = driver.find_element("id","email")
+    email.send_keys(mfp_username) #Email id 
+    password = driver.find_element("id","password")
+    password.send_keys(mfp_password)  #Password
+    time.sleep(3)
+    login = driver.find_element("id","password")
+    login.submit()   #Logging In
+    time.sleep(3)
+    cookies = driver.get_cookies()
+    put_cookies_in_jar(cookies,cj)
+
+    #----------- END Selenium workaround ---------------
+
     try:
-        client = myfitnesspal.Client(mfp_username,mfp_password)       
+        client = myfitnesspal.Client(cookiejar=cj) # 2022/08/29 Selenium workaround: addedd "cookiejar=cj".   
     except ValueError as e:
         with ErrorStdoutRedirection(ath_un):
             print((str(datetime.datetime.now()) + ' [' + sys._getframe().f_code.co_name + ']' + ' Error on line {}'.format(sys.exc_info()[-1].tb_lineno) + '-E1- ' + str(e)))
@@ -71,6 +132,8 @@ def dwnld_insert_nutrition(mfp_username,mfp_password,ath_un,start_date,end_date,
     with ProgressStdoutRedirection(ath_un):
         print('MFP Login successful! Proceeding...')
 
+    driver.quit() # 2022/08/29 Selenium workaround: Close the selenium session
+
     mfp_user_insert(mfp_username,encrypted_pwd,ath_un,db_host,db_name,superuser_un,superuser_pw,encr_pass) #PG: insert MFP user details into database
 
     # read DB connection parameters from ini file

src/mfp_data_download_db_insert.py

@@ -18,7 +18,7 @@ numpy
 dropbox==11.0.0
 Flask
 func_timeout==4.3.5
-myfitnesspal==1.16.6
+myfitnesspal==2.0.0
 mechanize==0.4.5
 dicttoxml==1.7.4
 psutil==5.8.0