Customized Scan Scope for Large Size Jira/Confluence/Asana Sites

requirements.txt

@@ -7,4 +7,5 @@ asana==3.2.2
 zenpy
 WrikePy
 BeautifulSoup4
-slack_sdk
+slack_sdk
+langchain-text-splitters

src/n0s1/__init__.py

@@ -1 +1 @@
-__version__ = "1.0.24"
+__version__ = "1.0.25"

src/n0s1/controllers/asana_controller.py

@@ -51,29 +51,121 @@ def is_connected(self):
                 self.log_message(f"Unable to connect to {self.get_name()} instance. Check your credentials.", logging.ERROR)
         return False
 
+    def _get_workspaces(self, limit=None):
+        workspaces = []
+        if self._scan_scope:
+            for key in self._scan_scope.get("workspaces", {}):
+                w = self._client.workspaces.get_workspace(key)
+                workspaces.append(w)
+        if len(workspaces) > 0:
+            return workspaces
+        self.connect()
+        return self._client.workspaces.get_workspaces()
+
+    def _get_projects(self, workspace_gid, limit=None):
+        projects = []
+        if self._scan_scope:
+            for key in self._scan_scope.get("workspaces", {}).get(workspace_gid, {}).get("projects", {}):
+                p = self._client.projects.get_project(key)
+                projects.append(p)
+        if len(projects) > 0:
+            return projects
+        self.connect()
+        return self._client.projects.get_projects_for_workspace(workspace_gid)
+
+    def _get_tasks(self, workspace_gid, project_gid, limit=None):
+        tasks = []
+        if self._scan_scope:
+            for key in self._scan_scope.get("workspaces", {}).get(workspace_gid, {}).get("projects", {}).get(project_gid, {}).get("tasks", {}):
+                t = self._client.tasks.get_task(key, opt_fields=["name", "gid", "notes", "permalink_url"])
+                tasks.append(t)
+        if len(tasks) > 0:
+            return tasks
+        self.connect()
+        return self._client.tasks.get_tasks_for_project(project_gid, opt_fields=["name", "gid", "notes", "permalink_url"])
+
+    def _get_stories(self, workspace_gid, project_gid, task_gid, limit=None):
+        stories = []
+        if self._scan_scope:
+            for key in self._scan_scope.get("workspaces", {}).get(workspace_gid, {}).get("projects", {}).get(project_gid, {}).get("tasks", {}).get(task_gid, {}).get("stories", {}):
+                s = self._client.stories.get_story(key)
+                stories.append(s)
+        if stories:
+            return stories
+        self.connect()
+        return self._client.stories.get_stories_for_task(task_gid)
+
+    def get_mapping(self, levels=-1, limit=None):
+        if not self._client:
+            return {}
+        map_data = {"workspaces": {}}
+        if workspaces := self._get_workspaces(limit):
+            for w in workspaces:
+                workspace_gid = w.get("gid", "")
+                if len(workspace_gid) > 0:
+                    w_item = {
+                        "gid": workspace_gid,
+                        "name": w.get("name", ""),
+                        "projects": {}
+                    }
+                    map_data["workspaces"][workspace_gid] = w_item
+                    if levels > 0 and levels <= 1:
+                        continue
+                    if projects := self._get_projects(workspace_gid, limit):
+                        for p in projects:
+                            project_gid = p.get("gid", "")
+                            p_item = {
+                                "gid": project_gid,
+                                "name": p.get("name", ""),
+                                "tasks": {}
+                            }
+                            if len(project_gid) > 0:
+                                map_data["workspaces"][workspace_gid]["projects"][project_gid] = p_item
+                            if levels > 0 and levels <= 2:
+                                continue
+                            tasks = self._get_tasks(workspace_gid, project_gid)
+                            for t in tasks:
+                                task_gid = t.get("gid", "")
+                                t_item = {
+                                    "gid": task_gid,
+                                    "name": t.get("name", ""),
+                                    "stories": {}
+                                }
+                                if len(project_gid) > 0:
+                                    map_data["workspaces"][workspace_gid]["projects"][project_gid]["tasks"][task_gid] = t_item
+                                if levels > 0 and levels <= 3:
+                                    continue
+                                stories = self._get_stories(workspace_gid, project_gid, task_gid, limit)
+                                for s in stories:
+                                    story_gid = s.get("gid", "")
+                                    s_item = {
+                                        "gid": story_gid,
+                                        "name": s.get("name", "")
+                                    }
+                                    if len(story_gid) > 0:
+                                        map_data["workspaces"][workspace_gid]["projects"][project_gid]["tasks"][task_gid]["stories"][story_gid] = s_item
+
+        return map_data
+
     def get_data(self, include_coments=False, limit=None):
         if not self._client:
             return {}
 
-        self.connect()
-        if workspaces := self._client.workspaces.get_workspaces():
+        if workspaces := self._get_workspaces(limit):
             for w in workspaces:
                 workspace_gid = w.get("gid", "")
-                self.connect()
-                if projects := self._client.projects.get_projects_for_workspace(workspace_gid):
+                if projects := self._get_projects(workspace_gid, limit):
                     for p in projects:
                         project_gid = p.get("gid", "")
-                        self.connect()
-                        if tasks := self._client.tasks.get_tasks_for_project(project_gid, opt_fields=["name", "gid", "notes", "permalink_url"]):
+                        if tasks := self._get_tasks(workspace_gid, project_gid, limit=None):
                             for t in tasks:
                                 comments = []
                                 title = t.get("name", "")
                                 task_gid = t.get("gid", "")
                                 description = t.get("notes", "")
                                 url = t.get("permalink_url", "")
                                 if include_coments:
-                                    self.connect()
-                                    if stories := self._client.stories.get_stories_for_task(task_gid):
+                                    if stories := self._get_stories( workspace_gid, project_gid, task_gid, limit):
                                         for s in stories:
                                             if s.get("type", "").lower() == "comment".lower():
                                                 comment = s.get("text", "")

src/n0s1/controllers/confluence_controller.py

@@ -121,51 +121,107 @@ def is_connected(self):
                 self.log_message(f"Unable to connect to {self.get_name()} instance. Check your credentials.", logging.ERROR)
         return False
 
-    def get_data(self, include_coments=False, limit=None):
-        from atlassian.confluence import ApiPermissionError
-        if not self._client:
-            return {}
+    def _get_workspaces(self, limit=None):
+        if self._scan_scope:
+            workspaces = []
+            for key in self._scan_scope.get("workspaces", {}):
+                workspaces.append(key)
+            yield workspaces
+        else:
+            space_start = 0
+            if not limit or limit < 0:
+                limit = 50
+            finished = False
+            while not finished:
+                try:
+                    self.connect()
+                    res = self._client.get_all_spaces(start=space_start, limit=limit)
+                    spaces = res.get("results", [])
+                except Exception as e:
+                    message = str(e) + f" get_all_spaces(start={space_start}, limit={limit})"
+                    self.log_message(message, logging.WARNING)
+                    spaces = [{}]
+                    time.sleep(1)
+                    continue
+                space_start += limit
+                if len(spaces) <= 0:
+                    finished = True
+                yield spaces
 
-        space_start = 0
+    def _get_pages(self, workspace_key, limit=None):
+        from atlassian.confluence import ApiPermissionError
+        start = 0
         if not limit or limit < 0:
             limit = 50
-        finished = False
-        while not finished:
-            try:
-                self.connect()
-                res = self._client.get_all_spaces(start=space_start, limit=limit)
-                spaces = res.get("results", [])
-            except Exception as e:
-                message = str(e) + f" get_all_spaces(start={space_start}, limit={limit})"
-                self.log_message(message, logging.WARNING)
-                spaces = [{}]
-                time.sleep(1)
-                continue
-            space_start += limit
+        pages_start = start
+        page_keys = []
+        if self._scan_scope:
+            page_keys = self._scan_scope.get("workspaces", {}).get(workspace_key, {})
+        if len(page_keys) > 0:
+            counter = 0
+            pages = []
+            for key in page_keys:
+                counter += 1
+                page = self._client.get_page_by_id(key)
+                pages.append(page)
+                if counter > limit:
+                    counter = 0
+                    yield pages
+                    pages = []
+            if len(pages) > 0:
+                yield pages
+        else:
+            if len(workspace_key) > 0:
+                pages_finished = False
+                while not pages_finished:
+                    try:
+                        self.connect()
+                        pages = self._client.get_all_pages_from_space(workspace_key, start=pages_start, limit=limit)
+                    except ApiPermissionError as e:
+                        message = str(e) + f" get_all_pages_from_space({workspace_key}, start={pages_start}, limit={limit}). Skipping..."
+                        self.log_message(message, logging.WARNING)
+                        pages = [{}]
+                        break
+                    except Exception as e:
+                        message = str(e) + f" get_all_pages_from_space({workspace_key}, start={pages_start}, limit={limit})"
+                        self.log_message(message, logging.WARNING)
+                        pages = [{}]
+                        time.sleep(1)
+                        continue
+                    pages_start += limit
+                    if len(pages) <= 0:
+                        pages_finished = True
+                    yield pages
+
+    def get_mapping(self, levels=-1, limit=None):
+        if not self._client:
+            return {}
+        map_data = {"workspaces": {}}
+        for spaces in self._get_workspaces(limit):
+            for space in spaces:
+                workspace_key = space.get("key", None)
+                if workspace_key:
+                    map_data["workspaces"][workspace_key] = {}
+                    if levels < 0 or levels > 1:
+                        for pages in self._get_pages(workspace_key, limit):
+                            for page in pages:
+                                page_id = page.get("id", None)
+                                page_title = page.get("title", "")
+                                if page_id:
+                                    map_data["workspaces"][workspace_key][page_id] = {"title": page_title}
+        return map_data
 
+    def get_data(self, include_coments=False, limit=None):
+        if not self._client:
+            return {}
+        for spaces in self._get_workspaces(limit):
             for s in spaces:
-                key = s.get("key", "")
+                key = s
+                if isinstance(s, dict):
+                    key = s.get("key", "")
                 self.log_message(f"Scanning Confluence space: [{key}]...")
                 if len(key) > 0:
-                    pages_start = 0
-                    pages_finished = False
-                    while not pages_finished:
-                        try:
-                            self.connect()
-                            pages = self._client.get_all_pages_from_space(key, start=pages_start, limit=limit)
-                        except ApiPermissionError as e:
-                            message = str(e) + f" get_all_pages_from_space({key}, start={pages_start}, limit={limit}). Skipping..."
-                            self.log_message(message, logging.WARNING)
-                            pages = [{}]
-                            break
-                        except Exception as e:
-                            message = str(e) + f" get_all_pages_from_space({key}, start={pages_start}, limit={limit})"
-                            self.log_message(message, logging.WARNING)
-                            pages = [{}]
-                            time.sleep(1)
-                            continue
-                        pages_start += limit
-
+                    for pages in self._get_pages(key, limit):
                         for p in pages:
                             comments = []
                             title = p.get("title", "")
@@ -183,6 +239,8 @@ def get_data(self, include_coments=False, limit=None):
                             description = body.get("body", {}).get("storage", {}).get("value", "")
                             url = body.get("_links", {}).get("base", "") + p.get("_links", {}).get("webui", "")
                             if len(page_id) > 0 and include_coments:
+                                if not limit or limit < 0:
+                                    limit = 50
                                 comments_start = 0
                                 comments_finished = False
                                 while not comments_finished:
@@ -208,12 +266,6 @@ def get_data(self, include_coments=False, limit=None):
                             ticket = self.pack_data(title, description, comments, url, page_id)
                             yield ticket
 
-                        if len(pages) <= 0:
-                            pages_finished = True
-
-            if len(spaces) <= 0:
-                finished = True
-
     def post_comment(self, issue, comment):
         if not self._client:
             return False

src/n0s1/controllers/hollow_controller.py

@@ -7,11 +7,13 @@ def __init__(self):
         self._config = None
         self._check_connection_after = 200
         self._requests_counter = 0
+        self._scan_scope = None
         self.log_message_callback = None
 
     def set_config(self, config=None):
         if config:
             self._config = config
+            self._scan_scope = self._config.get("scan_scope", None)
         return self._config is not None
 
     def set_log_message_callback(self, log_message_callback):
@@ -39,6 +41,9 @@ def is_connected(self):
     def get_data(self, include_coments=False, limit=None):
         return {}
 
+    def get_mapping(self, levels=-1, limit=None):
+        return {}
+
     def post_comment(self, issue, comment):
         return self.is_connected()