Fix Github Action

.github/workflows/python-flake.yml

@@ -1,7 +1,7 @@
 # This workflow will install Python dependencies, run tests and lint with a single version of Python
 # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
 
-name: Python flake
+name: Code Quality
 
 on: 
   pull_request:
@@ -29,10 +29,10 @@ jobs:
         pip install flake8
     - name: Lint with flake8
       run: |
-        # stop the build if there are Python syntax errors or undefined names
-        # flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        flake8 ./triplea/ --count --exit-zero --max-complexity=10 --format pylint --ignore F401,W503,W504,E203 --max-line-length=451 --statistics
+        flake8 ./triplea/  --count --exit-zero  --max-complexity=20 --max-line-length=90 --no-show-source  --statistics
+    - name: Fail if there are linting errors
+      if: ${{ failure() }}
+      run: exit 2
 
     # - name: Fetch action
     #   id: pl

.github/workflows/test-poetry-action.yml

@@ -1,6 +1,6 @@
 #https://github.com/marketplace/actions/install-poetry-action
 
-name: test-poetry-action
+name: Testing
 
 on: 
   pull_request:

CHANGELOG.md

@@ -3,6 +3,7 @@ All notable changes to this project will be documented in this file.
 
 ## v0.0.3 - 2023-09-27
 ### Improvements
+- Add `change_flag_extract_topic` in DAL
 - All Github Action For Build Package
 - Change go_extract_triple (Not Complete)
 - Add cli.visualize_file  for visual graph file
@@ -11,6 +12,7 @@ All notable changes to this project will be documented in this file.
 
 
 ### Bug Fixes
+- Fix Github Action
 - Fix language parsing in `export_triplea_csvs_in_relational_mode_save_file`
 - Remove spacy, transformers = {extras = ["torch"], version = "^4.30.0"}, pytextrank
 

README.md

@@ -19,11 +19,15 @@ This tool gives you the power to create a graph of articles and analyze it. This
 
 ![Build and push images](https://github.com/EhsanBitaraf/triple-a/workflows/push%20docker%20image/badge.svg)
 
+![Testing](https://github.com/EhsanBitaraf/triple-a/actions/workflows/test-poetry-action.yml)
+
+![Code Quality](https://github.com/EhsanBitaraf/triple-a/actions/workflows/python-flake.yml)
+
 <!-- ![Release](https://img.shields.io/github/release-date/EhsanBitaraf/triple-a) -->
 
-<!-- ![PyPI - Wheel](https://img.shields.io/pypi/EhsanBitaraf/triple-a) -->
+![PyPI - Wheel](https://img.shields.io/pypi/EhsanBitaraf/triple-a)
 
-<!-- [![GitHub commits](https://img.shields.io/github/commits-since/EhsanBitaraf/triple-a/v1.0.0.svg)](https://github.com/EhsanBitaraf/triple-a/commit/master) -->
+[![GitHub commits](https://img.shields.io/github/commits-since/EhsanBitaraf/triple-a/v1.0.0.svg)](https://github.com/EhsanBitaraf/triple-a/commit/master)
 
 
 

pyproject.toml

@@ -34,12 +34,18 @@ pymongo = "^4.3.3"
 py2neo = "^2021.2.3"
 ipykernel = "^6.21.2"
 ipywidgets = "^8.0.4"
+httpx = "^0.25.0"
+requests = "^2.31.0"
 
 
 [tool.poetry.group.dev.dependencies]
 black = "^23.1.0"
 pytest-cov = "^4.0.0"
 
+
+[tool.poetry.scripts]
+aaa = "triplea:cli.aaa"
+
 [build-system]
 requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"

triplea/client/topic_extraction.py

@@ -1,8 +1,79 @@
+import time
 from triplea.config.settings import SETTINGS
 import requests
 import json
 from triplea.service.click_logger import logger
 
+# import httpx
+# async def extract_topic(text: str,
+#                    method: str,
+#                    top:int=10,
+#                    threshold:float=0) -> list:
+
+
+#     URL  = SETTINGS.AAA_TOPIC_EXTRACT_ENDPOINT
+
+
+#     # data to be sent to api
+#     data = {
+#             "Text": text.replace("\n"," "),
+#             "Method": method,
+#             "Top": top,
+#             "Threshold": threshold
+#             }
+
+#     headers = {
+#         "User-Agent": SETTINGS.AAA_CLIENT_AGENT,
+#     }
+
+#     # To use HTTP Basic Auth with your proxy,
+#     #  use the http://user:password@host.com/ syntax:
+#     if SETTINGS.AAA_PROXY_HTTP is not None:
+#         proxy_servers = {
+#             "http": SETTINGS.AAA_PROXY_HTTP,
+#             "https": SETTINGS.AAA_PROXY_HTTPS,
+#         }
+#     else:
+#         proxy_servers = None
+
+#     # # sending get request and saving the response as response object
+#     # try:
+#     #     r = requests.post(url=URL,
+#     #                     data=json.dumps(data),
+#     #                     headers=headers,
+#     #                     proxies=proxy_servers)
+#     # except Exception:
+#     #     raise Exception("Connection Error.")
+
+
+    # async with httpx.AsyncClient() as client:
+    #     response = await client.post(url=URL,
+    #                     data=json.dumps(data),
+    #                     headers=headers,
+    #                     proxies=proxy_servers)
+    #     response.raise_for_status()
+
+    #     data =  response.json()
+    #     if 'status' in data:
+    #         return data['r']
+    #     else:
+    #         logger.ERROR('status not exist.')
+    #         raise
+
+    # # extracting data in json format
+    # try:
+    #     data = r.json()
+    #     if 'status' in data:
+    #         return data['r']
+    #     else:
+    #         logger.ERROR('status not exist.')
+    #         raise
+
+    # except Exception as ex:
+    #     logger.ERROR(f"Error : {ex}")
+    #     logger.ERROR(f"{type(r)}  {r} ")
+    #     raise
+
 
 def extract_topic(text: str,
                    method: str,
@@ -37,10 +108,12 @@ def extract_topic(text: str,
 
     # sending get request and saving the response as response object
     try:
+        j_data = json.dumps(data)
         r = requests.post(url=URL,
-                        data=json.dumps(data),
+                        data=j_data,
                         headers=headers,
                         proxies=proxy_servers)
+
     except Exception:
         raise Exception("Connection Error.")
 

triplea/db/mongodb.py

@@ -147,6 +147,15 @@ def get_article_group_by_state(self):
         ]
         return list(self.col_article.aggregate(pipeline))
 
+#region Extra Article Method
+
+    def change_flag_extract_topic(self,current_value,set_value):
+        myquery = {"FlagExtractTopic": current_value}
+        sett = {"$set": {"FlagExtractTopic": set_value}}
+        r = self.col_article.update_many(myquery, sett)
+        return r
+
+#endregion
 
 # endregion
 

triplea/db/tinydb.py

@@ -85,6 +85,15 @@ def get_all_article_count(self) -> int:
         """
         return len(self.db)
 
+
+#region Extra Article Method
+
+    def change_flag_extract_topic(self,current_value,set_value):
+        # Update the value of "FlagExtractTopic" from 0 to 1
+        return self.db.update({'FlagExtractTopic': set_value}, Query().FlagExtractTopic == current_value)
+
+#endregion
+
     def add_new_node(self, node: Node) -> int:
         node_json = json.loads(
             json.dumps(node, default=lambda o: o.__dict__, sort_keys=True, indent=4)

triplea/service/repository/persist.py

@@ -141,6 +141,14 @@ def get_article_group_by_state():
     """
     return db.get_article_group_by_state()
 
+#region Extra Article Method
+
+def change_flag_extract_topic(current_value,set_value):
+    return db.change_flag_extract_topic(current_value,set_value)
+
+#endregion
+
+
 #endregion
 
 #region Node

triplea/service/repository/pipeline_flag.py

@@ -106,8 +106,8 @@ def go_extract_triple():
                 logger.ERROR(f"Error {exc_tb}")
     persist.refresh()    
 
-def go_extract_topic(proccess_bar=False):
-    max_refresh_point = 500
+def go_extract_topic(proccess_bar=True):
+    max_refresh_point = 5
     l_pmid = persist.get_article_pmid_list_by_cstate( 0, "FlagExtractTopic" )
     total_article_in_current_state = len(l_pmid)
     n = 0

triplea/service/repository/state/custom/extract_topic.py

@@ -26,8 +26,7 @@ def extract_topic_abstract(article: Article):
     except:
         exc_type, exc_value, exc_tb = sys.exc_info()
         print()
-        print(exc_tb.tb_frame)
-        print()
+        # print(exc_tb.tb_frame)
 
         logger.ERROR(f"Error Line {exc_tb.tb_lineno}")
         logger.ERROR(f"Error {exc_value}")

triplea/the_private_backyard2.py

@@ -24,8 +24,10 @@
 
 if __name__ == "__main__":
     pass
-    export_triplea_csvs_in_relational_mode_save_file('export.csv',
-                                                     proccess_bar=True)
+    # export_triplea_csvs_in_relational_mode_save_file('export.csv',
+    #                                                  proccess_bar=True)
     # move_state_forward(2)
     # go_affiliation_mining()
-    # go_extract_topic(proccess_bar=True)
+
+    # persist.change_flag_extract_topic(1,0)
+    go_extract_topic(proccess_bar=True)