Merge pull request #97 from fossology/fix/content-disposition

- Add support for API version 1.4.0: adapt PUT and PATCH /upload
- Add possibility to specify `wait_time` for reports download
- Fix content disposition regex when downloading reports
- Create new minor version 1.5.0

Author: deveaud-m

.github/workflows/fossologytests.yml

@@ -53,7 +53,7 @@ jobs:
 
 
   test-last-release:
-    name: Integration Tests (Fossology 3.11.0)
+    name: Integration Tests (Fossology 4.0.0)
     runs-on: ubuntu-latest
 
     container:
@@ -63,7 +63,7 @@ jobs:
 
     services:
       fossology:
-        image: fossology/fossology:3.11.0
+        image: fossology/fossology:4.0.0
         ports: 
           - 8081:80
         volumes:

README.rst

@@ -26,7 +26,7 @@ See `the OpenAPI specification <https://raw.githubusercontent.com/fossology/foss
 
    - 1.2.1 (Fossology 3.10.0)
    - 1.3.2 (Fossology 3.11.0)
-   - 1.3.4 (next Fossology release)
+   - 1.4.0 (Fossology 4.0.0)
 
 Documentation
 =============

docs-source/conf.py

@@ -22,7 +22,7 @@
 copyright = "2021, Siemens AG"
 
 # The full version, including major/minor/patch tags
-release = "1.4.0"
+release = "1.5.0"
 
 
 # -- General configuration ---------------------------------------------------

fossology/__init__.py

@@ -283,7 +283,7 @@ def detail_user(self, user_id):
             raise FossologyApiError(description, response)
 
     def list_users(self):
-        """ List all users from the Fossology instance
+        """List all users from the Fossology instance
 
         API Endpoint: GET /users
 
@@ -389,7 +389,9 @@ def search(
             raise FossologyApiError(description, response)
 
     def filesearch(
-        self, filelist: List = [], group: str = None,
+        self,
+        filelist: List = [],
+        group: str = None,
     ):
         """Search for files from hash sum
 

fossology/foss_cli.py

@@ -222,7 +222,11 @@ def init_foss(ctx: dict):
     try:
         foss = Fossology(ctx.obj["SERVER"], ctx.obj["TOKEN"])  # using new API
     except AuthenticationError:  # API version < 1.2.3 requires a username
-        foss = Fossology(ctx.obj["SERVER"], ctx.obj["TOKEN"], name=ctx.obj["USERNAME"],)
+        foss = Fossology(
+            ctx.obj["SERVER"],
+            ctx.obj["TOKEN"],
+            name=ctx.obj["USERNAME"],
+        )
     ctx.obj["FOSS"] = foss
     ctx.obj["USER"] = foss.user.name
     logger.debug(f"Logged in as user {foss.user.name}")
@@ -233,7 +237,10 @@ def init_foss(ctx: dict):
 @click.group()
 @click.option("--token", "-t", help="The token to be used.")
 @click.option(
-    "--verbose", "-v", count=True, help="Increase verbosity level (e.g. -v -vv).",
+    "--verbose",
+    "-v",
+    count=True,
+    help="Increase verbosity level (e.g. -v -vv).",
 )
 @click.option(
     "--log_to_console/--no_log_to_console",
@@ -279,7 +286,7 @@ def cli(
     debug: bool,
     result_dir: str,
 ):
-    """The foss_cli cmdline.  Multiple -v increase verbosity-level. """
+    """The foss_cli cmdline.  Multiple -v increase verbosity-level."""
     if log_to_console:
         console_handler = logging.StreamHandler()
         console_handler.setFormatter(formatter)
@@ -443,12 +450,12 @@ def config(
 @click.pass_context
 def log(ctx: click.core.Context, log_level: int, message_text: str):
     """Add a Log Message to the log. If a log message is printed to the log depends
-       on the verbosity defined starting the foss_cli (default level 0 /-v level 1/-vv level 2).
-       Beeing on global verbosity level 0 only messages of --log_level 2 will be printed.
-       Beeing on global verbosity level 1 messages of --log_level 1 and 2 will be printed.
-       Beeing on global verbosity level 2 messages of --log_level 0,1,2 will be printed.
-       Where the log messages are printed depends on the global configuration for --log_to_console,
-       --log_to_file and --log_file_name.
+    on the verbosity defined starting the foss_cli (default level 0 /-v level 1/-vv level 2).
+    Beeing on global verbosity level 0 only messages of --log_level 2 will be printed.
+    Beeing on global verbosity level 1 messages of --log_level 1 and 2 will be printed.
+    Beeing on global verbosity level 2 messages of --log_level 0,1,2 will be printed.
+    Where the log messages are printed depends on the global configuration for --log_to_console,
+    --log_to_file and --log_file_name.
     """
 
     if log_level == 0:
@@ -514,7 +521,8 @@ def create_group(ctx: click.core.Context, group_name: str):
 
 @cli.command("upload_file")
 @click.argument(
-    "upload_file", type=click.Path(exists=True),
+    "upload_file",
+    type=click.Path(exists=True),
 )
 @click.option(
     "--folder_name", default="", show_default=True, help="The name of the upload folder"
@@ -596,7 +604,8 @@ def upload_file(
 
 @cli.command("start_workflow")
 @click.argument(
-    "file_name", type=click.Path(exists=True),
+    "file_name",
+    type=click.Path(exists=True),
 )
 @click.option(
     "--folder_name",

fossology/obj.py

@@ -311,7 +311,11 @@ class Findings(object):
     """
 
     def __init__(
-        self, scanner, conclusion, copyright=None, **kwargs,
+        self,
+        scanner,
+        conclusion,
+        copyright=None,
+        **kwargs,
     ):
         self.scanner = scanner
         self.conclusion = conclusion
@@ -478,7 +482,10 @@ class Licenses(object):
     """
 
     def __init__(
-        self, filePath, findings=None, **kwargs,
+        self,
+        filePath,
+        findings=None,
+        **kwargs,
     ):
         self.filepath = filePath
         if findings:
@@ -517,7 +524,12 @@ class Hash(object):
     """
 
     def __init__(
-        self, sha1, md5, sha256, size, **kwargs,
+        self,
+        sha1,
+        md5,
+        sha256,
+        size,
+        **kwargs,
     ):
         self.sha1 = sha1
         self.md5 = md5
@@ -548,7 +560,10 @@ class File(object):
     """
 
     def __init__(
-        self, hash, findings, **kwargs,
+        self,
+        hash,
+        findings,
+        **kwargs,
     ):
         self.hash = Hash.from_json(hash)
         self.findings = Findings.from_json(findings)

fossology/report.py

@@ -65,12 +65,21 @@ def generate_report(
             description = f"Report generation for upload {upload.uploadname} failed"
             raise FossologyApiError(description, response)
 
-    @retry(retry=retry_if_exception_type(TryAgain), stop=stop_after_attempt(3))
-    def download_report(self, report_id: int, group: str = None) -> Tuple[str, str]:
+    @retry(retry=retry_if_exception_type(TryAgain), stop=stop_after_attempt(10))
+    def download_report(
+        self, report_id: int, group: str = None, wait_time: int = 0
+    ) -> Tuple[str, str]:
         """Download a report
 
         API Endpoint: GET /report/{id}
 
+        Get report for a given upload. If the report is not ready wait another ``wait_time`` seconds or look at
+        the ``Retry-After`` to determine how long the wait period shall be.
+
+        If ``wait_time`` is 0, the time interval specified by the ``Retry-After`` header is used.
+
+        The function stops trying after **10 attempts**.
+
         :Example:
 
         >>> from fossology import Fossology
@@ -79,19 +88,22 @@ def download_report(self, report_id: int, group: str = None) -> Tuple[str, str]:
         >>>
         >>> # Generate a report for upload 1
         >>> report_id = foss.generate_report(foss.detail_upload(1)) # doctest: +SKIP
-        >>> report_content, report_name = foss.download_report(report_id) # doctest: +SKIP
+        >>> # Wait up to 20 Minutes until the report is ready
+        >>> report_content, report_name = foss.download_report(report_id, wait_time=120) # doctest: +SKIP
         >>> with open(report_name, "wb") as report_file:
         ...     report_file.write(report_content) # doctest: +SKIP
 
         :param report_id: the id of the generated report
         :param group: the group name to choose while downloading a specific report (default: None)
+        :param wait_time: use a customized upload wait time instead of Retry-After (in seconds, default: 0)
         :type report_id: int
         :type group: string
+        :type wait_time: int
         :return: the report content and the report name
         :rtype: Tuple[str, str]
         :raises FossologyApiError: if the REST call failed
         :raises AuthorizationError: if the user can't access the group
-        :raises TryAgain: if the report generation timed out after 3 retries
+        :raises TryAgain: if the report generation times out after 10 retries
         """
         headers = dict()
         if group:
@@ -100,17 +112,20 @@ def download_report(self, report_id: int, group: str = None) -> Tuple[str, str]:
         response = self.session.get(f"{self.api}/report/{report_id}", headers=headers)
         if response.status_code == 200:
             content = response.headers["Content-Disposition"]
-            report_name_pattern = '(^attachment; filename=")(.*)("$)'
-            report_name = re.match(report_name_pattern, content).group(2)
+            report_name_pattern = "(^attachment; filename=)(\"|')?([^\"|']*)(\"|'$)?"
+            report_name = re.match(report_name_pattern, content).group(3)
             return response.content, report_name
         elif response.status_code == 403:
             description = (
                 f"Getting report {report_id} {get_options(group)}not authorized"
             )
             raise AuthorizationError(description, response)
         elif response.status_code == 503:
-            wait_time = response.headers["Retry-After"]
-            logger.debug(f"Retry get report after {wait_time} seconds")
+            if not wait_time:
+                wait_time = response.headers["Retry-After"]
+            logger.debug(
+                f"Retry GET report {report_id} after {wait_time} seconds: {response.json()['message']}"
+            )
             time.sleep(int(wait_time))
             raise TryAgain
         else: