Merge 5f2a66f into 28a2010

Author: CoolCoderCarl

README.md

@@ -1,58 +1,47 @@
 # datapath
-
-Simple ETL getting info from https://newsapi.org/ and send to telegram channel
+Send info getting from API to telegram.  
+Third part of a system which search, save and share info.  
 
 ## Prehistory
-This simple service help to search information automatically via News API
+It is good idea to share info.
 
 > Datapath it is like telepath or astropath  
 > (c) Author
 
 Enjoy.
 
 ## How to use
-You can check the last available tags here - 
-1) https://hub.docker.com/repository/docker/h0d0user/truth_seeker
-2) https://hub.docker.com/repository/docker/h0d0user/datapath
-3) https://hub.docker.com/repository/docker/h0d0user/news_db
-
 Need to fill `settings.toml` with next important variables:
-1) `DB_NAME`. Where you want to load your data before send to telegram.
-2) `API_KEY`. You can find this data here - https://my.telegram.org/apps
-3) `QUERY`. Key word to search for in articles.
-4) `API_TOKEN`. Ask *BotFather* in telegram.
-5) `CHAT_ID`. Use this to find chat ID where you want to send messages - https://api.telegram.org/botAPI_TOKEN/getUpdates
-6) `docker-compose up -d`
-7) ...
-8) PROFIT !!!
-
-*It is not final configuration. You can find template below.*
-
-File `settings.toml` template:
+1) `API_KEY`. You can find this data here - https://my.telegram.org/apps
+2) `QUERY`. Key word to search for in articles.
+3) `API_TOKEN`. Ask *BotFather* in telegram.
+4) `CHAT_ID`. Use this to find chat ID where you want to send messages - https://api.telegram.org/botAPI_TOKEN/getUpdates
+5) `docker-compose up -d` or `make dstart`
+6) ...
+7) PROFIT !!!
+
 ```
 [DB]
-DB_NAME = "/mnt/test.db"
-
+DB_API_URL = "http://attainments_sanctuary:8888"
 
 [NEWS_API]
 API_KEY = ""
-QUERY = "test"
+QUERY = "computer science"
 LANGUAGE = "en"
 
-
 [TELEGRAM]
 API_TOKEN = ""
 CHAT_ID = ""
 
-
 [TIMINIGS]
-TIME_TO_PURGE = "00:00"
 TIME_TO_SEARCH = "02:00"
 TIME_TO_SEND_START = "10:00"
 TIME_TO_SEND_END = "20:00"
 SENDING_INTERVAL = 300
 ```
 
-
+For more info check:
+1) API & DB repo - https://github.com/CoolCoderCarl/attainments_sanctuary  
+2) ETL repo - https://github.com/CoolCoderCarl/epistolary
 
 **Still have questions ? Google it.**

datapath.py

@@ -37,6 +37,7 @@ def check_api_available() -> bool:
         return requests.get(f"{NEWS_DB_API_URL}/healthcheck").ok
     except (ConnectionError, ConnectionRefusedError) as con_err:
         logging.error(con_err)
+        return False
 
 
 def send_news_to_telegram(message):
@@ -73,23 +74,47 @@ def send_news_to_telegram(message):
         logging.error(err)
 
 
+def ask_entities() -> int:
+    """
+    Ask API request to discover how much entities in db
+    Return int instead of str
+    :return:
+    """
+    return int(requests.get(f"{NEWS_DB_API_URL}/entities").text)
+
+
 if __name__ == "__main__":
     while True:
         if check_api_available():
-            time.sleep(1)
             current_time = datetime.now().strftime("%H:%M")
             if TIME_TO_SEND_START < current_time < TIME_TO_SEND_END:
                 logging.info("Time to send news has come !")
-                data_from_db = requests.get(NEWS_DB_API_URL).json()
-                if len(data_from_db) == 0:
-                    logging.warning("Database is empty !")
+                if ask_entities() == 0:
+                    logging.warning("Database is empty ! Take a break for 30 min.")
+                    time.sleep(1800)
                 else:
-                    for news in data_from_db:
-                        send_news_to_telegram(news)
-                        time.sleep(SENDING_INTERVAL)
-                    else:
-                        logging.warning("All news was sent !")
+                    try:
+                        data_from_db = requests.get(f"{NEWS_DB_API_URL}/news").json()
+                        for news in data_from_db:
+                            send_news_to_telegram(news)
+                            time.sleep(SENDING_INTERVAL)
+                        else:
+                            logging.warning(
+                                f"All news was sent ! Going to purge ! Entities in db for now {ask_entities()}."
+                            )
+                            try:
+                                response = requests.get(f"{NEWS_DB_API_URL}/purge")
+                            except Exception as exception:
+                                logging.error(f"Exception while purging: {exception}.")
+                            else:
+                                logging.info(
+                                    f"Database was purged successfully ! Entities in db for now {ask_entities()}."
+                                )
+                    except Exception as exception:
+                        logging.error(f"Exception while getting news: {exception}")
             else:
                 logging.info("Still waiting to send.")
+                time.sleep(5)
         else:
             logging.error("API is not available !")
+            time.sleep(5)