feat/enhance jira connector (#493)

rbiseck3 · web-flow · commit 2306a7683365 · 2025-04-28T15:29:54.000-07:00
* support filtering by status

* support downloading attachments

* update changelog

* update fixtures

* drop unit test

* test attachment download in int test

* fix int tests

* use attachments data coming in from issue
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,6 +1,7 @@
-## 1.0.22-dev1
+## 1.0.23
 
 * **Migrate to new Mixedbread Python SDK**
+* **Support better filtering in jira connector and downloading attachments**
 
 ## 1.0.22
 
diff --git a/test/integration/connectors/expected_results/jira/directory_structure.json b/test/integration/connectors/expected_results/jira/directory_structure.json
@@ -4,6 +4,7 @@
     "JCTP1/10001.txt",
     "JCTP1/10002.txt",
     "JCTP1/10013.txt",
+    "JCTP1/attachments/10000.png",
     "JCTP2/10003.txt",
     "JCTP2/10004.txt",
     "JCTP2/10005.txt",
diff --git a/test/integration/connectors/expected_results/jira/downloads/JCTP1/attachments/10000.png b/test/integration/connectors/expected_results/jira/downloads/JCTP1/attachments/10000.png
diff --git a/test/integration/connectors/expected_results/jira/file_data/10000.json b/test/integration/connectors/expected_results/jira/file_data/10000.json
@@ -11,23 +11,19 @@
     "version": null,
     "record_locator": {
       "id": "10000",
-      "key": "JCTP1-1",
-      "board_id": null,
-      "project_id": "JCTP1"
+      "key": "JCTP1-1"
     },
     "date_created": "2023-08-22T11:29:37.774+0000",
     "date_modified": "2025-02-21T13:25:46.017+0000",
-    "date_processed": "1744387765.651267",
+    "date_processed": "1745877948.008202",
     "permissions_data": null,
     "filesize_bytes": null
   },
   "additional_metadata": {
     "id": "10000",
-    "key": "JCTP1-1",
-    "board_id": null,
-    "project_id": "JCTP1"
+    "key": "JCTP1-1"
   },
   "reprocess": false,
-  "local_download_path": "/private/var/folders/h7/n848df9s5yn7ml8rxb61vhyc0000gp/T/tmps1z7p7az/JCTP1/10000.txt",
+  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/tmphmteub8f/JCTP1/10000.txt",
   "display_name": "Jira Connector Test Project 1"
 }
diff --git a/test/integration/connectors/expected_results/jira/file_data/10000a.json b/test/integration/connectors/expected_results/jira/file_data/10000a.json
@@ -0,0 +1,47 @@
+{
+  "identifier": "10000a",
+  "connector_type": "jira",
+  "source_identifiers": {
+    "filename": "image-20230823-143650.png",
+    "fullpath": "10000/image-20230823-143650.png",
+    "rel_path": null
+  },
+  "metadata": {
+    "url": "https://unstructured-jira-connector-test.atlassian.net/rest/api/2/attachment/10000",
+    "version": null,
+    "record_locator": {
+      "id": "10000",
+      "filename": "image-20230823-143650.png",
+      "author": {
+        "self": "https://unstructured-jira-connector-test.atlassian.net/rest/api/2/user?accountId=712020%3A7bc7fdcb-67e7-435d-b4a2-128aee12820c",
+        "accountId": "712020:7bc7fdcb-67e7-435d-b4a2-128aee12820c",
+        "emailAddress": "devops+jira-connector@unstructured.io",
+        "avatarUrls": {
+          "48x48": "https://secure.gravatar.com/avatar/04abdb66f320c5c8cb47d9671ab8e20c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FUD-1.png",
+          "24x24": "https://secure.gravatar.com/avatar/04abdb66f320c5c8cb47d9671ab8e20c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FUD-1.png",
+          "16x16": "https://secure.gravatar.com/avatar/04abdb66f320c5c8cb47d9671ab8e20c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FUD-1.png",
+          "32x32": "https://secure.gravatar.com/avatar/04abdb66f320c5c8cb47d9671ab8e20c?d=https%3A%2F%2Favatar-management--avatars.us-west-2.prod.public.atl-paas.net%2Finitials%2FUD-1.png"
+        },
+        "displayName": "Unstructured Devops",
+        "active": true,
+        "timeZone": "Etc/GMT",
+        "accountType": "atlassian"
+      },
+      "mimeType": "image/png",
+      "content": "https://unstructured-jira-connector-test.atlassian.net/rest/api/2/attachment/content/10000",
+      "thumbnail": "https://unstructured-jira-connector-test.atlassian.net/rest/api/2/attachment/thumbnail/10000"
+    },
+    "date_created": "2023-08-23T14:37:07.577+0000",
+    "date_modified": "2025-02-21T13:25:46.017+0000",
+    "date_processed": "1745877948.008202",
+    "permissions_data": null,
+    "filesize_bytes": 516618
+  },
+  "additional_metadata": {
+    "id": "10000",
+    "key": "JCTP1-1"
+  },
+  "reprocess": false,
+  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/tmphmteub8f/JCTP1/attachments/10000.png",
+  "display_name": "Jira Connector Test Project 1"
+}
diff --git a/test/integration/connectors/expected_results/jira/file_data/10001.json b/test/integration/connectors/expected_results/jira/file_data/10001.json
@@ -11,23 +11,19 @@
     "version": null,
     "record_locator": {
       "id": "10001",
-      "key": "JCTP1-2",
-      "board_id": null,
-      "project_id": "JCTP1"
+      "key": "JCTP1-2"
     },
     "date_created": "2023-08-22T11:29:43.033+0000",
     "date_modified": "2023-08-24T12:03:31.591+0000",
-    "date_processed": "1744387766.059646",
+    "date_processed": "1745877950.036449",
     "permissions_data": null,
     "filesize_bytes": null
   },
   "additional_metadata": {
     "id": "10001",
-    "key": "JCTP1-2",
-    "board_id": null,
-    "project_id": "JCTP1"
+    "key": "JCTP1-2"
   },
   "reprocess": false,
-  "local_download_path": "/private/var/folders/h7/n848df9s5yn7ml8rxb61vhyc0000gp/T/tmps1z7p7az/JCTP1/10001.txt",
+  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/tmphmteub8f/JCTP1/10001.txt",
   "display_name": "Jira Connector Test Project 1"
 }
diff --git a/test/integration/connectors/expected_results/jira/file_data/10002.json b/test/integration/connectors/expected_results/jira/file_data/10002.json
@@ -11,23 +11,19 @@
     "version": null,
     "record_locator": {
       "id": "10002",
-      "key": "JCTP1-3",
-      "board_id": null,
-      "project_id": "JCTP1"
+      "key": "JCTP1-3"
     },
     "date_created": "2023-08-22T11:29:46.189+0000",
     "date_modified": "2023-08-23T14:36:31.252+0000",
-    "date_processed": "1744387766.421783",
+    "date_processed": "1745877950.495379",
     "permissions_data": null,
     "filesize_bytes": null
   },
   "additional_metadata": {
     "id": "10002",
-    "key": "JCTP1-3",
-    "board_id": null,
-    "project_id": "JCTP1"
+    "key": "JCTP1-3"
   },
   "reprocess": false,
-  "local_download_path": "/private/var/folders/h7/n848df9s5yn7ml8rxb61vhyc0000gp/T/tmps1z7p7az/JCTP1/10002.txt",
+  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/tmphmteub8f/JCTP1/10002.txt",
   "display_name": "Jira Connector Test Project 1"
 }
diff --git a/test/integration/connectors/expected_results/jira/file_data/10003.json b/test/integration/connectors/expected_results/jira/file_data/10003.json
@@ -11,23 +11,19 @@
     "version": null,
     "record_locator": {
       "id": "10003",
-      "key": "JCTP2-1",
-      "board_id": null,
-      "project_id": "JCTP2"
+      "key": "JCTP2-1"
     },
     "date_created": "2023-08-22T11:32:09.040+0000",
     "date_modified": "2023-09-29T05:55:11.447+0000",
-    "date_processed": "1744387767.135543",
+    "date_processed": "1745877946.192489",
     "permissions_data": null,
     "filesize_bytes": null
   },
   "additional_metadata": {
     "id": "10003",
-    "key": "JCTP2-1",
-    "board_id": null,
-    "project_id": "JCTP2"
+    "key": "JCTP2-1"
   },
   "reprocess": false,
-  "local_download_path": "/private/var/folders/h7/n848df9s5yn7ml8rxb61vhyc0000gp/T/tmps1z7p7az/JCTP2/10003.txt",
+  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/tmphmteub8f/JCTP2/10003.txt",
   "display_name": "Jira Connector Test Project 2"
 }
diff --git a/test/integration/connectors/expected_results/jira/file_data/10004.json b/test/integration/connectors/expected_results/jira/file_data/10004.json
@@ -11,23 +11,19 @@
     "version": null,
     "record_locator": {
       "id": "10004",
-      "key": "JCTP2-2",
-      "board_id": null,
-      "project_id": "JCTP2"
+      "key": "JCTP2-2"
     },
     "date_created": "2023-08-22T11:32:15.109+0000",
     "date_modified": "2023-09-29T05:55:11.351+0000",
-    "date_processed": "1744387767.493439",
+    "date_processed": "1745877946.676437",
     "permissions_data": null,
     "filesize_bytes": null
   },
   "additional_metadata": {
     "id": "10004",
-    "key": "JCTP2-2",
-    "board_id": null,
-    "project_id": "JCTP2"
+    "key": "JCTP2-2"
   },
   "reprocess": false,
-  "local_download_path": "/private/var/folders/h7/n848df9s5yn7ml8rxb61vhyc0000gp/T/tmps1z7p7az/JCTP2/10004.txt",
+  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/tmphmteub8f/JCTP2/10004.txt",
   "display_name": "Jira Connector Test Project 2"
 }
diff --git a/test/integration/connectors/expected_results/jira/file_data/10005.json b/test/integration/connectors/expected_results/jira/file_data/10005.json
@@ -11,23 +11,19 @@
     "version": null,
     "record_locator": {
       "id": "10005",
-      "key": "JCTP2-3",
-      "board_id": null,
-      "project_id": "JCTP2"
+      "key": "JCTP2-3"
     },
     "date_created": "2023-08-22T11:32:18.249+0000",
     "date_modified": "2023-08-22T11:32:21.717+0000",
-    "date_processed": "1744387767.834472",
+    "date_processed": "1745877947.087476",
     "permissions_data": null,
     "filesize_bytes": null
   },
   "additional_metadata": {
     "id": "10005",
-    "key": "JCTP2-3",
-    "board_id": null,
-    "project_id": "JCTP2"
+    "key": "JCTP2-3"
   },
   "reprocess": false,
-  "local_download_path": "/private/var/folders/h7/n848df9s5yn7ml8rxb61vhyc0000gp/T/tmps1z7p7az/JCTP2/10005.txt",
+  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/tmphmteub8f/JCTP2/10005.txt",
   "display_name": "Jira Connector Test Project 2"
 }
diff --git a/test/integration/connectors/expected_results/jira/file_data/10013.json b/test/integration/connectors/expected_results/jira/file_data/10013.json
@@ -11,23 +11,19 @@
     "version": null,
     "record_locator": {
       "id": "10013",
-      "key": "JCTP1-4",
-      "board_id": null,
-      "project_id": "JCTP1"
+      "key": "JCTP1-4"
     },
     "date_created": "2023-08-24T12:04:35.328+0000",
     "date_modified": "2023-08-24T12:04:47.543+0000",
-    "date_processed": "1744387766.776047",
+    "date_processed": "1745877950.902464",
     "permissions_data": null,
     "filesize_bytes": null
   },
   "additional_metadata": {
     "id": "10013",
-    "key": "JCTP1-4",
-    "board_id": null,
-    "project_id": "JCTP1"
+    "key": "JCTP1-4"
   },
   "reprocess": false,
-  "local_download_path": "/private/var/folders/h7/n848df9s5yn7ml8rxb61vhyc0000gp/T/tmps1z7p7az/JCTP1/10013.txt",
+  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/tmphmteub8f/JCTP1/10013.txt",
   "display_name": "Jira Connector Test Project 1"
 }
diff --git a/test/integration/connectors/expected_results/jira/file_data/10014.json b/test/integration/connectors/expected_results/jira/file_data/10014.json
@@ -11,23 +11,19 @@
     "version": null,
     "record_locator": {
       "id": "10014",
-      "key": "JCTP3-1",
-      "board_id": "3",
-      "project_id": "JCTP3"
+      "key": "JCTP3-1"
     },
     "date_created": "2023-08-24T13:38:35.057+0000",
     "date_modified": "2023-08-24T13:39:02.055+0000",
-    "date_processed": "1744387765.2460768",
+    "date_processed": "1745877944.5975628",
     "permissions_data": null,
     "filesize_bytes": null
   },
   "additional_metadata": {
     "id": "10014",
-    "key": "JCTP3-1",
-    "board_id": "3",
-    "project_id": "JCTP3"
+    "key": "JCTP3-1"
   },
   "reprocess": false,
-  "local_download_path": "/private/var/folders/h7/n848df9s5yn7ml8rxb61vhyc0000gp/T/tmps1z7p7az/JCTP3/10014.txt",
+  "local_download_path": "/private/var/folders/n8/rps3wl195pj4p_0vyxqj5jrw0000gn/T/tmphmteub8f/JCTP3/10014.txt",
   "display_name": "Jira Connector Test Project 3 - Company Managed Project"
 }
diff --git a/test/integration/connectors/test_jira.py b/test/integration/connectors/test_jira.py
@@ -43,7 +43,7 @@ async def test_jira_source(temp_dir):
     )
     index_config = JiraIndexerConfig(projects=projects, boards=boards, issues=issues)
 
-    download_config = JiraDownloaderConfig(download_dir=temp_dir)
+    download_config = JiraDownloaderConfig(download_dir=temp_dir, download_attachments=True)
 
     # Instantiate indexer and downloader
     indexer = JiraIndexer(
@@ -61,7 +61,7 @@ async def test_jira_source(temp_dir):
         downloader=downloader,
         configs=SourceValidationConfigs(
             test_id="jira",
-            expected_num_files=8,
+            expected_num_files=9,
             validate_file_data=True,
             validate_downloaded_files=True,
         ),
diff --git a/test/unit/connectors/test_jira.py b/test/unit/connectors/test_jira.py
diff --git a/unstructured_ingest/__version__.py b/unstructured_ingest/__version__.py
diff --git a/unstructured_ingest/processes/connectors/jira.py b/unstructured_ingest/processes/connectors/jira.py
