Merge pull request #17 from CoolCoderCarl/develop

Develop

Author: CoolCoderCarl

.dockerignore

@@ -6,6 +6,7 @@
 
 # Docker
 docker-compose.yml
+docker-compose.yaml
 **/Dockerfile
 **/Containerfile*
 .docker
@@ -14,4 +15,9 @@ docker-compose.yml
 # Meta
 README.md
 LICENSE
-requirements.txt
+
+# Configuration
+requirements.txt
+changelog.json
+settings.toml
+*.db

.github/workflows/publish.yaml

@@ -129,6 +129,5 @@ jobs:
             <b>Pushed image:</b> <code>docker pull h0d0user/news_db:${{ needs.prepare.outputs.get_current_tag }}</code>,
             <code>docker pull h0d0user/truth_seeker:${{ needs.prepare.outputs.get_current_tag }}</code>,
             <code>docker pull h0d0user/datapath:${{ needs.prepare.outputs.get_current_tag }}</code>
-            See changes: https://github.com/${{ github.repository }}/commit/${{ github.sha }}
 
             See changes: https://github.com/${{ github.repository }}/commit/${{ github.sha }}

README.md

@@ -17,13 +17,42 @@ You can check the last available tags here -
 3) https://hub.docker.com/repository/docker/h0d0user/news_db
 
 Need to fill `settings.toml` with next important variables:
-1) `API_KEY`. You can find this data here - https://my.telegram.org/apps
-2) `API_TOKEN`. Ask *BotFather* in telegram.
-3) `CHAT_ID`. Use this to find chat ID where you want to send messages - https://api.telegram.org/botAPI_TOKEN/getUpdates
-4) `DB_NAME`. Where you want to load your data before send to telegram.
-5) `QUERY`. Key word to search for in articles.
+1) `DB_NAME`. Where you want to load your data before send to telegram.
+2) `API_KEY`. You can find this data here - https://my.telegram.org/apps
+3) `QUERY`. Key word to search for in articles.
+4) `API_TOKEN`. Ask *BotFather* in telegram.
+5) `CHAT_ID`. Use this to find chat ID where you want to send messages - https://api.telegram.org/botAPI_TOKEN/getUpdates
 6) `docker-compose up -d`
 7) ...
 8) PROFIT !!!
 
+*It is not final configuration. You can find template below.*
+
+File `settings.toml` template:
+```
+[DB]
+DB_NAME = "/mnt/test.db"
+
+
+[NEWS_API]
+API_KEY = ""
+QUERY = "test"
+LANGUAGE = "en"
+
+
+[TELEGRAM]
+API_TOKEN = ""
+CHAT_ID = ""
+
+
+[TIMINIGS]
+TIME_TO_PURGE = "00:00"
+TIME_TO_SEARCH = "02:00"
+TIME_TO_SEND_START = "10:00"
+TIME_TO_SEND_END = "20:00"
+SENDING_INTERVAL = 300
+```
+
+
+
 **Still have questions ? Google it.**

datapath.py

@@ -70,14 +70,16 @@ def send_news_to_telegram(message):
             time.sleep(1)
             current_time = datetime.now().strftime("%H:%M")
             if TIME_TO_SEND_START < current_time < TIME_TO_SEND_END:
-                logging.info("Time to send has come !")
+                logging.info("Time to send news has come !")
                 data_from_db = news_db.send_all_news(db_connection)
                 if len(data_from_db) == 0:
                     logging.warning("Database is empty !")
                 else:
                     for news in data_from_db:
                         send_news_to_telegram(news)
                         time.sleep(SENDING_INTERVAL)
+                    else:
+                        logging.warning("All news was sent !")
             else:
                 logging.info("Still waiting to send.")
         else:

news_db.py

@@ -25,16 +25,20 @@
 INSERT_INTO_SQL = """
 INSERT INTO news
 (author,title,description,url,pub_date)
-VALUES(?,?,?,?,?)
+VALUES(?,?,?,?,?);
 """
 
 
 SELECT_FROM_SQL = """
-SELECT * FROM news
+SELECT * FROM news;
 """
 
 DELETE_FROM_SQL = """
-DELETE FROM news
+DELETE FROM news;
+"""
+
+SELECT_COUNT_SQL = """
+SELECT COUNT(*) FROM news;
 """
 
 
@@ -47,6 +51,10 @@
 )
 
 
+def check_entities_count(conn):
+    return conn.cursor().execute(SELECT_COUNT_SQL).fetchone()[0]
+
+
 def create_connection(db_file: Path):
     """
     Create db file
@@ -71,7 +79,7 @@ def create_table(conn, create_table_query):
     try:
         c = conn.cursor()
         c.execute(create_table_query)
-        logging.info("Table created successfully !")
+        logging.info(f"Table created successfully !")
     except Error as create_table_err:
         logging.error(create_table_err)
 
@@ -88,7 +96,9 @@ def insert_into(conn, data: tuple):
         cur = conn.cursor()
         cur.execute(INSERT_INTO_SQL, data)
         conn.commit()
-        logging.info("Data inserted successfully !")
+        logging.info(
+            f"Data inserted successfully ! Entities in db for now: {check_entities_count(conn)}"
+        )
         return cur.lastrowid
     except Error as insert_err:
         logging.error(insert_err)
@@ -100,12 +110,7 @@ def send_all_news(conn):
     :param conn: Connection to the SQLite database
     :return:
     """
-    cur = conn.cursor()
-    cur.execute(SELECT_FROM_SQL)
-
-    rows = cur.fetchall()
-
-    return rows
+    return conn.cursor().execute(SELECT_FROM_SQL).fetchall()
 
 
 def delete_all_news(conn):
@@ -114,10 +119,11 @@ def delete_all_news(conn):
     :param conn: Connection to the SQLite database
     :return:
     """
-    cur = conn.cursor()
-    cur.execute(DELETE_FROM_SQL)
+    conn.cursor().execute(DELETE_FROM_SQL)
     conn.commit()
-    logging.info("Database was purged !")
+    logging.info(
+        f"Database was purged ! Entities in db for now: {check_entities_count(conn)}"
+    )
 
 
 if __name__ == "__main__":
@@ -132,9 +138,13 @@ def delete_all_news(conn):
             if conn is not None:
                 send_all_news(conn)
                 if CURRENT_TIME == TIME_TO_PURGE:
-                    logging.info("Time to purge has come !")
+                    logging.info(
+                        f"Time to purge has come ! Entities in db for now: {check_entities_count(conn)}"
+                    )
                     delete_all_news(conn)
                 else:
-                    logging.info("Still waiting for purging.")
+                    logging.info(
+                        f"Still waiting for purging. Entities in db for now: {check_entities_count(conn)}"
+                    )
     except sqlite3.Error as sql_err:
         logging.error(f"Cannot create the database connection. Error: {sql_err}")

truth_seeker.py

@@ -73,7 +73,8 @@ def load_to_db(fetch_info: dict):
                     article_list.append(article_data)
             news_db.insert_into(
                 news_db.create_connection(news_db.DB_FILE),
-                tuple(article_list),
+                # Pull too much info but add set conversion
+                tuple(set(article_list)),
             )
     else:
         logging.warning("Empty response from News API.")
@@ -84,7 +85,6 @@ def load_to_db(fetch_info: dict):
     while True:
         CURRENT_TIME = datetime.now().strftime("%H:%M")
         time.sleep(1)
-        # Pull too much info
         if CURRENT_TIME == TIME_TO_SEARCH:
             logging.info("Time to search has come !")
             try: