diff --git a/pyproject.toml b/pyproject.toml index 0c4e31e..08201db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,6 +77,9 @@ select = [ "B", # flake8-bugbear "SIM", # flake8-simplify ] +ignore = [ + "E501", +] [tool.pytest.ini_options] testpaths = [ @@ -85,3 +88,6 @@ testpaths = [ addopts = [ "--import-mode=importlib", ] +markers = [ + "slow: marks tests as slow (deselect with '-m \"not slow\"')", +] diff --git a/src/onc/modules/_DataProductFile.py b/src/onc/modules/_DataProductFile.py index b3d1e23..16b11fa 100644 --- a/src/onc/modules/_DataProductFile.py +++ b/src/onc/modules/_DataProductFile.py @@ -15,11 +15,23 @@ def __init__(self, max_retries): class _DataProductFile: """ - Donwloads a single data product file + Downloads a single data product file Is able to poll and wait if required """ - def __init__(self, dpRunId: int, index: str, baseUrl: str, token: str): + def __init__( + self, + dpRunId: int, + index: str, + baseUrl: str, + token: str, + verbosity: str = "INFO", + ): + self.verbosity = verbosity + # Use child logger 'onc.poll' consistent with _PollLog + from ._Messages import setup_logger + + self._log = setup_logger("onc.poll", verbosity) self._retries = 0 self._status = 202 self._downloaded = False @@ -50,7 +62,7 @@ def download( Can poll, wait and retry if the file is not ready to download Return the file information """ - log = _PollLog(True) + log = _PollLog(self.verbosity) self._status = 202 while self._status == 202: # Run timed request @@ -74,9 +86,9 @@ def download( response, outPath, filename, overwrite ) except FileExistsError: - if self._retries > 1: - print("") - print(f' Skipping "{self._filePath}": File already exists.') + self._log.info( + f' Skipping "{self._filePath}": File already exists.' + ) self._status = 777 elif self._status == 202: # Still processing, wait and retry @@ -84,10 +96,9 @@ def download( sleep(pollPeriod) elif self._status == 204: # No data found - print(" No data found.") + self._log.info(" No data found.") elif self._status == 404: # Index too high, no more files to download - log.printNewLine() pass elif self._status == 410: # Status 410: gone (file deleted from FTP) @@ -98,7 +109,8 @@ def download( stacklevel=2, ) else: - raise requests.HTTPError(_createErrorMessage(response)) + self._log.error(_createErrorMessage(response)) + response.raise_for_status() return self._status diff --git a/src/onc/modules/_Messages.py b/src/onc/modules/_Messages.py new file mode 100644 index 0000000..821625c --- /dev/null +++ b/src/onc/modules/_Messages.py @@ -0,0 +1,139 @@ +import logging +import re +import time + +import requests + +REQ_MSG = "Requested: {}" # get request url +RESPONSE_TIME_MSG = "Response received in {} seconds." # requests.elapsed value. +RESPONSE_MSG = "HTTP Response: {} ({})" # Brief description, status code +MULTIPAGE_MSG = ( + "The requested data quantity is greater than the " + "supplied row limit and will be downloaded over multiple requests." +) + + +LEVEL_MAP = { + "CRITICAL": logging.CRITICAL, + "ERROR": logging.ERROR, + "WARNING": logging.WARNING, + "INFO": logging.INFO, + "DEBUG": logging.DEBUG, +} + + +class OnclibFormatter(logging.Formatter): + """ + Custom formatter that removes prefix for INFO level logs. + """ + + def format(self, record): + if record.levelno == logging.INFO: + return record.getMessage() + return super().format(record) + + +def setup_logger(logger_name: str = "onc", level: int | str = "INFO") -> logging.Logger: + """ + Set up a logger object for displaying verbose messages to console. + + Parameters + ---------- + logger_name : str, optional + The unique logger name to use. Can be shared between modules. + level : int or str, optional + The logging level to use. Default is 'INFO'. + + Returns + ------- + logging.Logger + The configured logging.Logger object. + """ + + # Ensure level is a valid logging level integer + if isinstance(level, str): + level = LEVEL_MAP.get(level.upper(), logging.INFO) + + logger = logging.getLogger(logger_name) + logger.propagate = False + + # Apply level to the logger itself + logger.setLevel(level) + + if not logger.handlers: + console = logging.StreamHandler() + console.setLevel(level) + + # Set the logging format. + dtfmt = "%Y-%m-%dT%H:%M:%S" + strfmt = ( + "%(asctime)s.%(msecs)03dZ | %(name)-12s | %(levelname)-8s | %(message)s" + ) + fmt = OnclibFormatter(strfmt, datefmt=dtfmt) + fmt.converter = time.gmtime + + console.setFormatter(fmt) + logger.addHandler(console) + else: + # If handlers exist, ensure they have the correct level + for handler in logger.handlers: + handler.setLevel(level) + + return logger + + +def scrub_token(input: str) -> str: + """ + Replace a token in a query URL or other string with the string 'REDACTED' + so that users don't accidentally commit their tokens to public repositories + if ONC Info/Warnings are too verbose. + + Parameters + ---------- + input : str + An Oceans 3.0 API URL or string with a token query parameter. + + Returns + ------- + str + A scrubbed url. + """ + return re.sub(r"([?&]token=)[a-f0-9-]{36}", r"\1REDACTED", input) + + +def build_error_message(response: requests.Response, redact_token: bool) -> str: + """ + Build an error message from a requests.Response object. + + Parameters + ---------- + response : requests.Response + A requests.Response object. + redact_token : bool + If true, redact tokens before returning an error message. + + Returns + ------- + str + An error message. + """ + payload = response.json() + message = payload.get("message") + + if "errors" in payload: + errors = payload["errors"] + error_messages = [] + for error in errors: + emsg = ( + f"(API Error Code {error['errorCode']}) " + f"{error['errorMessage']} for query parameter(s) " + f"'{error['parameter']}'." + ) + error_messages.append(emsg) + error_message = "\n".join(error_messages) + else: + error_message = None + msg = "\n".join([m for m in (message, error_message) if m is not None]) + if redact_token is True and "token=" in msg: + msg = scrub_token(msg) + return msg diff --git a/src/onc/modules/_MultiPage.py b/src/onc/modules/_MultiPage.py index 6369773..af458de 100644 --- a/src/onc/modules/_MultiPage.py +++ b/src/onc/modules/_MultiPage.py @@ -4,7 +4,9 @@ from time import time import dateutil.parser -import humanize +from onc.modules._Messages import ( + setup_logger, +) from ._util import _formatDuration @@ -14,12 +16,25 @@ class _MultiPage: def __init__(self, parent: object): self.parent = weakref.ref(parent) self.result = None + self._log = setup_logger("onc.multi", self._config("verbosity") or "INFO") + + def _config(self, key): + p = self.parent() + if p is None: + return None + if hasattr(p, "_config"): + return p._config(key) + return getattr(p, key, None) def getAllPages(self, service: str, url: str, filters: dict): """ Requests all pages from the service, with the url and filters Multiple pages will be downloaded until completed - @return: Service response with concatenated data for all pages obtained + + Returns + ------- + dict + Service response with concatenated data for all pages obtained. """ # pop archivefiles extension extension = None @@ -30,48 +45,54 @@ def getAllPages(self, service: str, url: str, filters: dict): # download first page start = time() response, responseTime = self._doPageRequest(url, filters, service, extension) - rNext = response["next"] - - if rNext is not None: - print( - "Data quantity is greater than the row limit and", - "will be downloaded in multiple pages.", - ) - - pageCount = 1 - pageEstimate = self._estimatePages(response, service) - if pageEstimate > 0: - # Exclude the first page when calculating the time estimation - timeEstimate = _formatDuration((pageEstimate - 1) * responseTime) - print( - f"Downloading time for the first page: {humanize.naturaldelta(responseTime)}" # noqa: E501 - ) - print(f"Estimated approx. {pageEstimate} pages in total.") - print( - f"Estimated approx. {timeEstimate} to complete for the rest of the pages." # noqa: E501 - ) - # keep downloading pages until next is None - print("") - while rNext is not None: - pageCount += 1 - rowCount = self._rowCount(response, service) + if isinstance(response, dict): + rNext = response["next"] - print(f" ({rowCount} samples) Downloading page {pageCount}...") - nextResponse, nextTime = self._doPageRequest( - url, rNext["parameters"], service, extension + if rNext is not None: + self._log.info( + "The requested data quantity is greater than the supplied " + "row limit and will be downloaded over multiple requests." ) - rNext = nextResponse["next"] - - # concatenate new data obtained - self._catenateData(response, nextResponse, service) - totalTime = _formatDuration(time() - start) - print( - f" ({self._rowCount(response, service):d} samples)" - f" Completed in {totalTime}." - ) - response["next"] = None + pageCount = 1 + pageEstimate = self._estimatePages(response, service) + if pageEstimate > 0: + # Exclude the first page when calculating the time estimation + timeEstimate = _formatDuration((pageEstimate - 1) * responseTime) + self._log.info( + f"Download time for page {pageCount}: {round(responseTime, 2)} seconds" + ) + self._log.info( + f"Est. number of pages remaining for download: {pageEstimate - 1}" + ) + self._log.info( + f"Est. number of seconds to download remaining data: {timeEstimate}" + ) + + # keep downloading pages until next is None + while rNext is not None: + pageCount += 1 + rowCount = self._rowCount(response, service) + + self._log.info( + f" Submitting request for page {pageCount} ({rowCount} samples)..." + ) + + nextResponse, nextTime = self._doPageRequest( + url, rNext["parameters"], service, extension + ) + rNext = nextResponse["next"] + + # concatenate new data obtained + self._catenateData(response, nextResponse, service) + + totalTime = _formatDuration(time() - start) + + self._log.info( + f" Downloaded {self._rowCount(response, service):d} total samples in {totalTime}." + ) + response["next"] = None return response @@ -81,8 +102,22 @@ def _doPageRequest( """ Wraps the _doRequest method Performs additional processing of the response for certain services - @param extension: Only provide for archivefiles filtering - Returns a tuple (jsonResponse, duration) + + Parameters + ---------- + url : str + API endpoint URL. + filters : dict + Filters for the request. + service : str + Name of the service (e.g. archivefile). + extension : str, optional + Only provide for archivefiles filtering. + + Returns + ------- + tuple + (jsonResponse, duration) """ if service.startswith("archivefile"): response, duration = self.parent()._doRequest(url, filters, getTime=True) diff --git a/src/onc/modules/_OncArchive.py b/src/onc/modules/_OncArchive.py index 9129a99..759e061 100644 --- a/src/onc/modules/_OncArchive.py +++ b/src/onc/modules/_OncArchive.py @@ -6,7 +6,7 @@ from ._MultiPage import _MultiPage from ._OncService import _OncService -from ._util import _createErrorMessage, _formatDuration, saveAsFile +from ._util import _formatDuration, saveAsFile class _OncArchive(_OncService): @@ -73,8 +73,10 @@ def downloadArchivefile(self, filename: str = "", overwrite: bool = False): size, downloadTime = saveAsFile(response, outPath, filename, overwrite) else: - msg = _createErrorMessage(response) - raise requests.HTTPError(msg) + from ._Messages import build_error_message + + self._log.error(build_error_message(response, self._config("redact_token"))) + response.raise_for_status() # Prepare a readable status txtStatus = "error" @@ -109,7 +111,7 @@ def downloadDirectArchivefile( dataRows = self.getArchivefile(filters, allPages) n = len(dataRows["files"]) - print(f"Obtained a list of {n} files to download.") + self._log.info(f"Obtained a list of {n} files to download.") # Download the files obtained tries = 1 @@ -124,7 +126,7 @@ def downloadDirectArchivefile( fileExists = os.path.exists(filePath) if not fileExists or os.path.getsize(filePath) == 0 or overwrite: - print(f' ({tries} of {n}) Downloading file: "{filename}"') + self._log.info(f' ({tries} of {n}) Downloading file: "{filename}"') downInfo = self.downloadArchivefile(filename, overwrite) size += downInfo["size"] time += downInfo["downloadTime"] @@ -132,7 +134,7 @@ def downloadDirectArchivefile( successes += 1 tries += 1 else: - print(f' Skipping "{filename}": File already exists.') + self._log.info(f' Skipping "{filename}": File already exists.') downInfo = { "url": self.getArchivefileUrl(filename), "status": "skipped", @@ -142,8 +144,8 @@ def downloadDirectArchivefile( } downInfos.append(downInfo) - print(f"{successes} files ({humanize.naturalsize(size)}) downloaded") - print(f"Total Download Time: {_formatDuration(time)}") + self._log.info(f"{successes} files ({humanize.naturalsize(size)}) downloaded") + self._log.info(f"Total Download Time: {_formatDuration(time)}") return { "downloadResults": downInfos, diff --git a/src/onc/modules/_OncDelivery.py b/src/onc/modules/_OncDelivery.py index 616ce24..3889b94 100644 --- a/src/onc/modules/_OncDelivery.py +++ b/src/onc/modules/_OncDelivery.py @@ -8,7 +8,7 @@ from ._DataProductFile import _DataProductFile from ._OncService import _OncService from ._PollLog import _PollLog -from ._util import _createErrorMessage, _formatSize +from ._util import _formatSize class _OncDelivery(_OncService): @@ -56,7 +56,6 @@ def orderDataProduct( ) ) - print("") self._printProductOrderStats(fileList, runData) return self._formatResult(fileList, runData) @@ -88,9 +87,9 @@ def runDataProduct(self, dpRequestId: int, waitComplete: bool): Return a dictionary with information of the run process. """ status = "" - log = _PollLog(True) - print( - f"To cancel the running data product, run 'onc.cancelDataProduct({dpRequestId})'" # noqa: E501 + log = _PollLog(self._config("verbosity")) + self._log.info( + f"To cancel the running data product, run 'onc.cancelDataProduct({dpRequestId})'" ) url = f"{self._config('baseUrl')}api/dataProductDelivery/run" runResult = {"runIds": [], "fileCount": 0, "runTime": 0, "requestCount": 0} @@ -111,7 +110,12 @@ def runDataProduct(self, dpRequestId: int, waitComplete: bool): if response.ok: data = response.json() else: - raise requests.HTTPError(_createErrorMessage(response)) + from ._Messages import build_error_message + + self._log.error( + build_error_message(response, self._config("redact_token")) + ) + response.raise_for_status() if waitComplete: status = data[0]["status"] @@ -128,10 +132,6 @@ def runDataProduct(self, dpRequestId: int, waitComplete: bool): runResult["fileCount"] = data[0]["fileCount"] runResult["runTime"] = time() - start - # print a new line after the process finishes - if waitComplete: - print("") - # gather a list of runIds for run in data: runResult["runIds"].append(run["dpRunId"]) @@ -192,9 +192,11 @@ def _downloadProductFiles( # keep increasing index until fileCount or until we get 404 doLoop = True timeout = self._config("timeout") - print(f"\nDownloading data product files with runId {runId}...") + self._log.info(f"Downloading data product files with runId {runId}...") - dpf = _DataProductFile(runId, str(index), baseUrl, token) + dpf = _DataProductFile( + runId, str(index), baseUrl, token, self._config("verbosity") + ) # loop thorugh file indexes while doLoop: @@ -211,7 +213,9 @@ def _downloadProductFiles( # file was downloaded (200), or skipped before downloading (777) fileList.append(dpf.getInfo()) index += 1 - dpf = _DataProductFile(runId, str(index), baseUrl, token) + dpf = _DataProductFile( + runId, str(index), baseUrl, token, self._config("verbosity") + ) elif status != 202 or (fileCount > 0 and index >= fileCount): # no more files to download @@ -219,7 +223,9 @@ def _downloadProductFiles( # get metadata if required if getMetadata: - dpf = _DataProductFile(runId, "meta", baseUrl, token) + dpf = _DataProductFile( + runId, "meta", baseUrl, token, self._config("verbosity") + ) try: status = dpf.download( timeout, @@ -248,8 +254,8 @@ def _infoForProductFiles(self, dpRunId: int, fileCount: int, getMetadata: bool): Returned rows will have the same structure as those returned by _DataProductFile.getInfo(). """ - print( - f"\nObtaining download information for data product files with runId {dpRunId}..." # noqa: E501 + self._log.info( + f"Obtaining download information for data product files with runId {dpRunId}..." ) # If we don't know the fileCount, get it from the server (takes longer) @@ -268,6 +274,7 @@ def _infoForProductFiles(self, dpRunId: int, fileCount: int, getMetadata: bool): index=str(index), baseUrl=self._config("baseUrl"), token=self._config("token"), + verbosity=self._config("verbosity"), ) dpf.setComplete() fileList.append(dpf.getInfo()) @@ -304,7 +311,7 @@ def _countFilesInProduct(self, runId: int): filters["index"] += 1 n += 1 - print(f" {n} files available for download") + self._log.info(f" {n} files available for download") return n def _printProductRequest(self, response): @@ -315,19 +322,19 @@ def _printProductRequest(self, response): product source (archive or generated on the fly). """ isGenerated = "estimatedFileSize" in response - print(f"Request Id: {response['dpRequestId']}") + self._log.info(f"Request Id: {response['dpRequestId']}") if isGenerated: size = response["estimatedFileSize"] # API returns it as a formatted string - print(f"Estimated File Size: {size}") + self._log.info(f"Estimated File Size: {size}") if "estimatedProcessingTime" in response: - print( + self._log.info( f"Estimated Processing Time: {response['estimatedProcessingTime']}" ) else: size = _formatSize(response["fileSize"]) - print(f"File Size: {size}") - print("Data product is ready for download.") + self._log.info(f"File Size: {size}") + self._log.info("Data product is ready for download.") def _estimatePollPeriod(self, response): """ @@ -373,7 +380,7 @@ def _printProductOrderStats(self, fileList: list, runInfo: dict): # Print run time runTime = timedelta(seconds=runInfo["runTime"]) - print(f"Total run time: {humanize.naturaldelta(runTime)}") + self._log.info(f"Total run time: {humanize.naturaldelta(runTime)}") if downloadCount > 0: # Print download time @@ -381,13 +388,13 @@ def _printProductOrderStats(self, fileList: list, runInfo: dict): txtDownTime = f"{downloadTime:.3f} seconds" else: txtDownTime = humanize.naturaldelta(downloadTime) - print(f"Total download Time: {txtDownTime}") + self._log.info(f"Total download Time: {txtDownTime}") # Print size and count of files natural_size = humanize.naturalsize(size, binary=True) - print(f"{downloadCount} files ({natural_size}) downloaded") + self._log.info(f"{downloadCount} files ({natural_size}) downloaded") else: - print("No files downloaded.") + self._log.info("No files downloaded.") def _formatResult(self, fileList: list, runInfo: dict): size = 0 diff --git a/src/onc/modules/_OncRealTime.py b/src/onc/modules/_OncRealTime.py index 5ec55cb..7ce93ec 100644 --- a/src/onc/modules/_OncRealTime.py +++ b/src/onc/modules/_OncRealTime.py @@ -9,8 +9,8 @@ class _OncRealTime(_OncService): Near real-time services methods """ - def __init__(self, config: dict): - super().__init__(config) + def __init__(self, parent: object): + super().__init__(parent) def getScalardataByLocation(self, filters: dict, allPages: bool): """ diff --git a/src/onc/modules/_OncService.py b/src/onc/modules/_OncService.py index 02eb532..39392dc 100644 --- a/src/onc/modules/_OncService.py +++ b/src/onc/modules/_OncService.py @@ -1,14 +1,14 @@ -import logging -import pprint import weakref -from time import time -from urllib import parse import requests - -from ._util import _createErrorMessage, _formatDuration - -logging.basicConfig(format="%(levelname)s: %(message)s") +from onc.modules._Messages import ( + REQ_MSG, + RESPONSE_MSG, + RESPONSE_TIME_MSG, + build_error_message, + scrub_token, + setup_logger, +) class _OncService: @@ -18,6 +18,7 @@ class _OncService: def __init__(self, parent: object): self.parent = weakref.ref(parent) + self._log = setup_logger("onc.service", level=self._config("verbosity")) def _doRequest(self, url: str, filters: dict | None = None, getTime: bool = False): """ @@ -44,48 +45,44 @@ def _doRequest(self, url: str, filters: dict | None = None, getTime: bool = Fals filters["token"] = self._config("token") timeout = self._config("timeout") - txtParams = parse.unquote(parse.urlencode(filters)) - self._log(f"Requesting URL:\n{url}?{txtParams}") - - start = time() response = requests.get(url, filters, timeout=timeout) - responseTime = time() - start - if response.ok: - jsonResult = response.json() + if self._config("redact_token"): + response_url = scrub_token(response.url) else: - status = response.status_code - if status in [400, 401]: - msg = _createErrorMessage(response) - raise requests.HTTPError(msg) - else: - response.raise_for_status() - self._log(f"Web Service response time: {_formatDuration(responseTime)}") - - # Log warning messages only when showWarning is True - # and jsonResult["messages"] is not an empty list - if ( - self._config("showWarning") - and "messages" in jsonResult - and jsonResult["messages"] - ): - long_message = "\n".join( - [f"* {message}" for message in jsonResult["messages"]] - ) + response_url = response.url - filters_without_token = filters.copy() - del filters_without_token["token"] - filters_str = pprint.pformat(filters_without_token) + # Log the url the user submitted. + self._log.debug(REQ_MSG.format(response_url)) - logging.warning( - f"When calling {url} with filters\n{filters_str},\n" - f"there are several warning messages:\n{long_message}\n" - ) + # Display the time it took for ONC to respond in seconds. + # The requests.Response.elapsed value is a datetime.timedelta object. + responseTime = round(response.elapsed.total_seconds(), 3) # To milliseconds. + self._log.debug(RESPONSE_TIME_MSG.format(responseTime)) + + json_response = response.json() + + # Log warning messages if json_response["messages"] is not an empty list + if "messages" in json_response and json_response["messages"]: + for message in json_response["messages"]: + self._log.warning(f"* {message}") - if getTime: - return jsonResult, responseTime + if response.status_code in (requests.codes.ok, requests.codes.accepted): + self._log.debug(RESPONSE_MSG.format(response.reason, response.status_code)) + if getTime: + return json_response, responseTime + return json_response else: - return jsonResult + self._log.error(RESPONSE_MSG.format(response.reason, response.status_code)) + + self._log.error(build_error_message(response, self._config("redact_token"))) + + if self._config("raise_http_errors"): + response.raise_for_status() + + if getTime: + return response, responseTime + return response def _serviceUrl(self, service: str): """ @@ -115,14 +112,6 @@ def _serviceUrl(self, service: str): return "" - def _log(self, message: str): - """ - Prints message to console only when self.showInfo is true - @param message: String - """ - if self._config("showInfo"): - print(message) - def _config(self, key: str): """ Returns a property from the parent (ONC class) diff --git a/src/onc/modules/_PollLog.py b/src/onc/modules/_PollLog.py index 1d32620..e7595cb 100644 --- a/src/onc/modules/_PollLog.py +++ b/src/onc/modules/_PollLog.py @@ -1,24 +1,30 @@ +from onc.modules._Messages import setup_logger + + class _PollLog: """ A helper for DataProductFile Keeps track of the messages printed in a single product download process """ - def __init__(self, showInfo: bool): + def __init__(self, verbosity: str): """ - @param showInfo same as in parent ONC object + Parameters + ---------- + verbosity : str + Standard logging level string. """ self._messages = [] # unique messages returned during the product order self._runStart = 0.0 # {float} timestamp (seconds) self._runEnd = 0.0 - self._showInfo = showInfo # flag for writing console messages - self._doPrintFileCount = True - self._lastPrintedDot = False # True after printing a dot (.) without a newline + self._verbosity = verbosity + self._log = setup_logger("onc.poll", level=verbosity) + self._doLogFileCount = True def logMessage(self, response): """ Adds a message to the messages list if it's new - Prints message to console, or "." if it repeats itself + Logs message to logger """ # Detect if the response comes from a "run" or "download" method origin = "download" @@ -35,33 +41,20 @@ def logMessage(self, response): # Detect and print change in the file count if origin == "run": fileCount = response[0]["fileCount"] - if self._doPrintFileCount and fileCount > 0: - self.printInfo( - f"\n {fileCount} files generated for this data product", - True, + if self._doLogFileCount and fileCount > 0: + self._log.info( + f" {fileCount} files generated for this data product" ) - self._doPrintFileCount = False + self._doLogFileCount = False self._messages.append(msg) - self.printInfo("\n " + msg, sameLine=True) + self._log.info(" " + msg) else: - self.printInfo(".", sameLine=True) - - def printInfo(self, msg: str, sameLine: bool = False): - """ - Conditional printing helper - """ - self._lastPrintedDot = msg == "." - - if self._showInfo: - if sameLine: - print(msg, end="", flush=True) - else: - print(msg) + # For repeating messages, log at DEBUG level to avoid cluttering INFO logs + self._log.debug(msg) def printNewLine(self): """ - Prints a line break only if the last message printed was a dot (.) + Deprecated. Newlines are handled by the logger. """ - if self._lastPrintedDot: - print("") + pass diff --git a/src/onc/modules/_util.py b/src/onc/modules/_util.py index 16db3fb..87bae0e 100644 --- a/src/onc/modules/_util.py +++ b/src/onc/modules/_util.py @@ -35,7 +35,11 @@ def saveAsFile( def _formatSize(size: float) -> str: """ Returns a formatted file size string representation - @param size: {float} Size in bytes + + Parameters + ---------- + size : float + Size in bytes. """ return humanize.naturalsize(size) @@ -43,7 +47,11 @@ def _formatSize(size: float) -> str: def _formatDuration(secs: float) -> str: """ Returns a formatted time duration string representation of a duration in seconds - @param seconds: float + + Parameters + ---------- + secs : float + Duration in seconds. """ if secs < 1.0: txtDownTime = f"{secs:.3f} seconds" diff --git a/src/onc/onc.py b/src/onc/onc.py index 42b1e85..2950019 100644 --- a/src/onc/onc.py +++ b/src/onc/onc.py @@ -1,10 +1,12 @@ import datetime import json +import logging import os import re from pathlib import Path from dateutil import parser +from onc.modules._Messages import setup_logger from onc.modules._OncArchive import _OncArchive from onc.modules._OncDelivery import _OncDelivery from onc.modules._OncDiscovery import _OncDiscovery @@ -27,14 +29,12 @@ class ONC: - True: Use the production server. - False: Use the internal ONC test server (reserved for ONC staff IP addresses). - showInfo : boolean, default False - Whether verbose script messages are displayed, such as request url and processing time information. - - - True: Print all information and debug messages (intended for debugging). - - False: Only print information messages. - showWarning : boolean, default True - Whether warning messages are displayed. Some web services have "messages" key in the response JSON - to indicate that something might need attention, like using a default value for a missing parameter. + verbosity : str, default "INFO" + The logging verbosity level. Supported values: "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL". + showInfo : boolean [Deprecated] + Use verbosity="DEBUG" or "INFO" instead. + showWarning : boolean [Deprecated] + Use verbosity="WARNING" instead. outPath : str | Path, default "output" The directory that files are saved to (relative to the current directory) when downloading files. The directory will be created if it does not exist during the download. @@ -46,18 +46,34 @@ class ONC: >>> from onc import ONC >>> onc = ONC() # Only if you set the env variable "ONC_TOKEN" # doctest: +SKIP >>> onc = ONC("YOUR_TOKEN_HERE") # doctest: +SKIP - >>> onc = ONC("YOUR_TOKEN_HERE", showInfo=True, outPath="onc-files") # doctest: +SKIP + >>> onc = ONC("YOUR_TOKEN_HERE", verbosity="DEBUG", outPath="onc-files") # doctest: +SKIP """ # noqa: E501 def __init__( self, token: str | None = None, - production: bool = True, - showInfo: bool = False, - showWarning: bool = True, outPath: str | Path = "output", + verbosity: str = "INFO", + raise_http_errors: bool = True, + redact_token: bool = False, timeout: int = 60, + production: bool = True, + **kwargs, ): + if "showInfo" in kwargs or "showWarning" in kwargs: + import warnings + + warnings.warn( + "showInfo and showWarning are deprecated. Use verbosity instead.", + DeprecationWarning, + stacklevel=2, + ) + # If verbosity wasn't explicitly changed from default, use showInfo/showWarning to set it + if verbosity == "INFO": + if kwargs.get("showInfo") is True: + verbosity = "DEBUG" + if kwargs.get("showWarning") is False: + verbosity = "ERROR" if token is None or token == "": token = os.environ.get("ONC_TOKEN") if token is None or token == "": @@ -65,9 +81,14 @@ def __init__( "ONC API token is required. Please provide it as the first argument, " "or set it as the environment variable 'ONC_TOKEN'." ) + + self.verbosity = verbosity + self.redact_token = redact_token + self.raise_http_errors = raise_http_errors + self._log = setup_logger("onc", self.verbosity) + self.token = re.sub(r"[^a-zA-Z0-9\-]+", "", token) - self.showInfo = showInfo - self.showWarning = showWarning + self.timeout = timeout self.production = production self.outPath = outPath @@ -78,6 +99,8 @@ def __init__( self.realTime = _OncRealTime(self) self.archive = _OncArchive(self) + self._log.debug("Initialized ONC module.") + @property def outPath(self) -> Path: """ @@ -107,6 +130,28 @@ def production(self, is_production: bool) -> None: else: self.baseUrl = "https://qa.oceannetworks.ca/" + @property + def verbosity(self) -> str: + """ + Return the current verbosity level. + """ + return self._verbosity + + @verbosity.setter + def verbosity(self, level: str) -> None: + """ + Set the logging verbosity level for the ONC instance and all its services. + + Supported values: "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL". + """ + self._verbosity = level.upper() + setup_logger("onc", self._verbosity) + + # Update all existing loggers that start with 'onc.' + for name in logging.root.manager.loggerDict: + if name.startswith("onc."): + setup_logger(name, self._verbosity) + def print(self, obj, filename: str = "") -> None: """ Pretty print a collection to the console or a file. @@ -138,6 +183,34 @@ def print(self, obj, filename: str = "") -> None: with open(filePath, "w+") as file: file.write(text) + @property + def showInfo(self) -> bool: + """ + [Deprecated] Return whether the logger level is DEBUG. + """ + return self._log.getEffectiveLevel() <= logging.DEBUG + + @showInfo.setter + def showInfo(self, value: bool) -> None: + """ + [Deprecated] Set the logger level based on the boolean value. + """ + self.verbosity = "DEBUG" if value else "INFO" + + @property + def showWarning(self) -> bool: + """ + [Deprecated] Return whether the logger level is WARNING or lower. + """ + return self._log.getEffectiveLevel() <= logging.WARNING + + @showWarning.setter + def showWarning(self, value: bool) -> None: + """ + [Deprecated] Set the logger level based on the boolean value. + """ + self.verbosity = "WARNING" if value else "ERROR" + def formatUtc(self, dateString: str = "now") -> str: """ Format the provided date string as an ISO8601 UTC date string. diff --git a/src/onc/util/util.py b/src/onc/util/util.py index b66b5e8..f407837 100644 --- a/src/onc/util/util.py +++ b/src/onc/util/util.py @@ -341,9 +341,14 @@ def copyFieldIfExists(fromDic, toDic, keys): """ Copy the field at name from fromDic to toDic only if it exists - @param fromDic: Origin Dictionary - @param toDic: Destination Dictionary - @param keys: Array of keys of the elements to copy + Parameters + ---------- + fromDic : dict + Origin Dictionary. + toDic : dict + Destination Dictionary. + keys : list + Array of keys of the elements to copy. """ for key in keys: if key in fromDic: diff --git a/tests/archive_file/test_archivefile_device.py b/tests/archive_file/test_archivefile_device.py index f6ebdc0..a589f9b 100644 --- a/tests/archive_file/test_archivefile_device.py +++ b/tests/archive_file/test_archivefile_device.py @@ -20,9 +20,9 @@ def params_multiple_pages(params): return params | {"rowLimit": 2} -def test_invalid_param_value(requester, params): +def test_invalid_param_value(requester, params, err_400): params_invalid_param_value = params | {"deviceCode": "XYZ123"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getArchivefile(params_invalid_param_value) @@ -32,9 +32,9 @@ def test_invalid_params_missing_required(requester, params): requester.getArchivefile(params) -def test_invalid_param_name(requester, params): +def test_invalid_param_name(requester, params, err_400): params_invalid_param_name = params | {"deviceCodes": "BPR-Folger-59"} - with pytest.raises(requests.HTTPError, match=r"API Error 129"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getArchivefile(params_invalid_param_name) diff --git a/tests/archive_file/test_archivefile_download.py b/tests/archive_file/test_archivefile_download.py index 8ae6c9b..ef0900b 100644 --- a/tests/archive_file/test_archivefile_download.py +++ b/tests/archive_file/test_archivefile_download.py @@ -4,13 +4,13 @@ import requests -def test_invalid_param_value(requester): - with pytest.raises(requests.HTTPError, match=r"API Error 96"): +def test_invalid_param_value(requester, err_400): + with pytest.raises(requests.HTTPError, match=err_400): requester.downloadArchivefile("FAKEFILE.XYZ") -def test_invalid_params_missing_required(requester): - with pytest.raises(requests.HTTPError, match=r"API Error 128"): +def test_invalid_params_missing_required(requester, err_400): + with pytest.raises(requests.HTTPError, match=err_400): requester.downloadArchivefile() diff --git a/tests/archive_file/test_archivefile_location.py b/tests/archive_file/test_archivefile_location.py index 89fb1b6..9900b21 100644 --- a/tests/archive_file/test_archivefile_location.py +++ b/tests/archive_file/test_archivefile_location.py @@ -2,9 +2,9 @@ import requests -def test_invalid_param_value(requester, params_location): +def test_invalid_param_value(requester, params_location, err_400): params_invalid_param_value = params_location | {"locationCode": "XYZ123"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getArchivefile(params_invalid_param_value) @@ -14,18 +14,18 @@ def test_invalid_params_missing_required(requester, params_location): requester.getArchivefile(params_location) -def test_invalid_param_name(requester, params_location): +def test_invalid_param_name(requester, params_location, err_400): params_invalid_param_name = params_location | {"locationCodes": "NCBC"} - with pytest.raises(requests.HTTPError, match=r"API Error 129"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getArchivefile(params_invalid_param_name) -def test_no_data(requester, params_location): +def test_no_data(requester, params_location, err_400): params_no_data = params_location | { "dateFrom": "2000-01-01", "dateTo": "2000-01-02", } - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getArchivefile(params_no_data) diff --git a/tests/conftest.py b/tests/conftest.py index 12d434d..3cdb286 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,6 +2,7 @@ from pathlib import Path import pytest +import requests from dotenv import load_dotenv from onc import ONC @@ -19,6 +20,16 @@ def requester(tmp_path) -> ONC: return ONC(production=is_prod, outPath=tmp_path) +@pytest.fixture +def err_400(): + return f"{requests.codes.bad} Client Error" + + +@pytest.fixture +def err_404(): + return f"{requests.codes.not_found} Client Error" + + @pytest.fixture(scope="session") def util(): return Util diff --git a/tests/data_product_delivery/test_data_product_delivery_cancel.py b/tests/data_product_delivery/test_data_product_delivery_cancel.py index ab347b7..f68b19e 100644 --- a/tests/data_product_delivery/test_data_product_delivery_cancel.py +++ b/tests/data_product_delivery/test_data_product_delivery_cancel.py @@ -2,6 +2,6 @@ import requests -def test_invalid_request_id(requester): - with pytest.raises(requests.HTTPError, match=r"API Error 127"): +def test_invalid_request_id(requester, err_400): + with pytest.raises(requests.HTTPError, match=err_400): requester.cancelDataProduct(1234567890) diff --git a/tests/data_product_delivery/test_data_product_delivery_download.py b/tests/data_product_delivery/test_data_product_delivery_download.py index faacce4..14c000a 100644 --- a/tests/data_product_delivery/test_data_product_delivery_download.py +++ b/tests/data_product_delivery/test_data_product_delivery_download.py @@ -2,6 +2,6 @@ import requests -def test_invalid_run_id(requester): - with pytest.raises(requests.HTTPError, match=r"API Error 127"): +def test_invalid_run_id(requester, err_400): + with pytest.raises(requests.HTTPError, match=err_400): requester.downloadDataProduct(1234567890) diff --git a/tests/data_product_delivery/test_data_product_delivery_order.py b/tests/data_product_delivery/test_data_product_delivery_order.py index 7b16df2..06a6524 100644 --- a/tests/data_product_delivery/test_data_product_delivery_order.py +++ b/tests/data_product_delivery/test_data_product_delivery_order.py @@ -2,12 +2,13 @@ import requests -def test_invalid_param_value(requester, params): +def test_invalid_param_value(requester, params, err_400): params_invalid_param_value = params | {"dataProductCode": "XYZ123"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.orderDataProduct(params_invalid_param_value) +@pytest.mark.slow def test_valid_default(requester, params, expected_keys_download_results, util): data = requester.orderDataProduct(params) @@ -30,6 +31,7 @@ def test_valid_default(requester, params, expected_keys_download_results, util): ) +@pytest.mark.slow def test_valid_no_metadata(requester, params, expected_keys_download_results, util): data = requester.orderDataProduct(params, includeMetadataFile=False) @@ -49,6 +51,7 @@ def test_valid_no_metadata(requester, params, expected_keys_download_results, ut ) +@pytest.mark.slow def test_valid_results_only(requester, params, expected_keys_download_results, util): data = requester.orderDataProduct(params, downloadResultsOnly=True) diff --git a/tests/data_product_delivery/test_data_product_delivery_restart.py b/tests/data_product_delivery/test_data_product_delivery_restart.py index 4cf1fe5..672ad59 100644 --- a/tests/data_product_delivery/test_data_product_delivery_restart.py +++ b/tests/data_product_delivery/test_data_product_delivery_restart.py @@ -2,6 +2,6 @@ import requests -def test_invalid_request_id(requester): - with pytest.raises(requests.HTTPError, match=r"API Error 127"): +def test_invalid_request_id(requester, err_400): + with pytest.raises(requests.HTTPError, match=err_400): requester.restartDataProduct(1234567890) diff --git a/tests/data_product_delivery/test_data_product_delivery_run.py b/tests/data_product_delivery/test_data_product_delivery_run.py index 76463c1..0e3dcdc 100644 --- a/tests/data_product_delivery/test_data_product_delivery_run.py +++ b/tests/data_product_delivery/test_data_product_delivery_run.py @@ -2,6 +2,6 @@ import requests -def test_invalid_request_id(requester): - with pytest.raises(requests.HTTPError, match=r"API Error 127"): +def test_invalid_request_id(requester, err_400): + with pytest.raises(requests.HTTPError, match=err_400): requester.runDataProduct(1234567890) diff --git a/tests/data_product_delivery/test_data_product_delivery_status.py b/tests/data_product_delivery/test_data_product_delivery_status.py index a5eb452..4a48aec 100644 --- a/tests/data_product_delivery/test_data_product_delivery_status.py +++ b/tests/data_product_delivery/test_data_product_delivery_status.py @@ -2,6 +2,6 @@ import requests -def test_invalid_request_id(requester): - with pytest.raises(requests.HTTPError, match=r"API Error 127"): +def test_invalid_request_id(requester, err_400): + with pytest.raises(requests.HTTPError, match=err_400): requester.checkDataProduct(1234567890) diff --git a/tests/data_product_delivery/test_integration.py b/tests/data_product_delivery/test_integration.py index 39014c7..1c67ee5 100644 --- a/tests/data_product_delivery/test_integration.py +++ b/tests/data_product_delivery/test_integration.py @@ -1,6 +1,7 @@ import pytest +@pytest.mark.slow def test_valid_manual(requester, params, expected_keys_download_results, util): """ Test request -> status -> run -> download -> status. @@ -35,6 +36,7 @@ def test_valid_manual(requester, params, expected_keys_download_results, util): assert data_status_after_download["searchHdrStatus"] == "COMPLETED" +@pytest.mark.slow def test_valid_cancel_restart(requester, params, expected_keys_download_results, util): """ Test request -> run -> cancel -> download (fail) -> restart -> download. diff --git a/tests/discover_data_availability/test_data_availability.py b/tests/discover_data_availability/test_data_availability.py index 1a4a201..a748707 100644 --- a/tests/discover_data_availability/test_data_availability.py +++ b/tests/discover_data_availability/test_data_availability.py @@ -12,9 +12,9 @@ def params(): } -def test_invalid_param_value(requester, params): +def test_invalid_param_value(requester, params, err_400): params_invalid_param_value = params | {"locationCode": "INVALID"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getDataAvailability(params_invalid_param_value) diff --git a/tests/discover_data_products/test_data_products.py b/tests/discover_data_products/test_data_products.py index 6d149bc..8d75281 100644 --- a/tests/discover_data_products/test_data_products.py +++ b/tests/discover_data_products/test_data_products.py @@ -2,25 +2,25 @@ import requests -def test_invalid_param_value(requester): +def test_invalid_param_value(requester, err_400): params_invalid_param_value = {"dataProductCode": "XYZ123"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getDataProducts(params_invalid_param_value) -def test_invalid_param_name(requester): +def test_invalid_param_name(requester, err_400): params_invalid_param_name = {"dataProductCodes": "HSD"} - with pytest.raises(requests.HTTPError, match=r"API Error 129"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getDataProducts(params_invalid_param_name) -def test_no_data(requester): +def test_no_data(requester, err_404): params_no_data = { "dataProductCode": "HSD", "extension": "txt", } - with pytest.raises(requests.HTTPError, match=r"404 Client Error"): + with pytest.raises(requests.HTTPError, match=err_404): requester.getDataProducts(params_no_data) diff --git a/tests/discover_deployments/test_deployments.py b/tests/discover_deployments/test_deployments.py index b984978..76b3b30 100644 --- a/tests/discover_deployments/test_deployments.py +++ b/tests/discover_deployments/test_deployments.py @@ -2,46 +2,46 @@ import requests -def test_invalid_time_range_greater_start_time(requester): +def test_invalid_time_range_greater_start_time(requester, err_400): params_invalid_time_range_greater_start_time = { "locationCode": "BACAX", "deviceCategoryCode": "CTD", "dateFrom": "2020-01-01", "dateTo": "2019-01-01", } - with pytest.raises(requests.HTTPError, match=r"API Error 23"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getDeployments(params_invalid_time_range_greater_start_time) -def test_invalid_time_range_future_start_time(requester): +def test_invalid_time_range_future_start_time(requester, err_400): params_invalid_time_range_future_start_time = { "locationCode": "BACAX", "deviceCategoryCode": "CTD", "dateFrom": "2050-01-01", } - with pytest.raises(requests.HTTPError, match=r"API Error 25"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getDeployments(params_invalid_time_range_future_start_time) -def test_invalid_param_value(requester): +def test_invalid_param_value(requester, err_400): params_invalid_param_value = {"locationCode": "XYZ123", "deviceCategoryCode": "CTD"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getDeployments(params_invalid_param_value) -def test_invalid_param_name(requester): +def test_invalid_param_name(requester, err_400): params_invalid_param_name = {"locationCodes": "BACAX", "deviceCategoryCode": "CTD"} - with pytest.raises(requests.HTTPError, match=r"API Error 129"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getDeployments(params_invalid_param_name) -def test_no_data(requester): +def test_no_data(requester, err_400): params_no_data = { "locationCode": "BACAX", "deviceCategoryCode": "CTD", "dateTo": "1900-01-01", } - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getDeployments(params_no_data) diff --git a/tests/discover_device_categories/test_device_categories.py b/tests/discover_device_categories/test_device_categories.py index 45a2a0f..3a39445 100644 --- a/tests/discover_device_categories/test_device_categories.py +++ b/tests/discover_device_categories/test_device_categories.py @@ -2,25 +2,25 @@ import requests -def test_invalid_param_value(requester): +def test_invalid_param_value(requester, err_400): params_invalid_param_value = {"deviceCategoryCode": "XYZ123"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getDeviceCategories(params_invalid_param_value) -def test_invalid_param_name(requester): +def test_invalid_param_name(requester, err_400): params_invalid_param_name = {"deviceCategoryCodes": "CTD"} - with pytest.raises(requests.HTTPError, match=r"API Error 129"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getDeviceCategories(params_invalid_param_name) -def test_no_data(requester): +def test_no_data(requester, err_404): params_no_data = { "deviceCategoryCode": "CTD", "deviceCategoryName": "Conductivity", "description": "TemperatureXXX", } - with pytest.raises(requests.HTTPError, match=r"404 Client Error"): + with pytest.raises(requests.HTTPError, match=err_404): requester.getDeviceCategories(params_no_data) diff --git a/tests/discover_devices/test_devices.py b/tests/discover_devices/test_devices.py index 96069aa..1ffa135 100644 --- a/tests/discover_devices/test_devices.py +++ b/tests/discover_devices/test_devices.py @@ -2,40 +2,40 @@ import requests -def test_invalid_time_range_greater_start_time(requester): +def test_invalid_time_range_greater_start_time(requester, err_400): params_invalid_time_range_greater_start_time = { "deviceCode": "BPR-Folger-59", "dateFrom": "2020-01-01", "dateTo": "2019-01-01", } - with pytest.raises(requests.HTTPError, match=r"API Error 23"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getDevices(params_invalid_time_range_greater_start_time) -def test_invalid_time_range_future_start_time(requester): +def test_invalid_time_range_future_start_time(requester, err_400): params_invalid_time_range_future_start_time = { "deviceCode": "BPR-Folger-59", "dateFrom": "2050-01-01", } - with pytest.raises(requests.HTTPError, match=r"API Error 25"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getDevices(params_invalid_time_range_future_start_time) -def test_invalid_param_value(requester): +def test_invalid_param_value(requester, err_400): params_invalid_param_value = {"deviceCode": "XYZ123"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getDevices(params_invalid_param_value) -def test_invalid_param_name(requester): +def test_invalid_param_name(requester, err_400): params_invalid_param_name = {"deviceCodes": "BPR-Folger-59"} - with pytest.raises(requests.HTTPError, match=r"API Error 129"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getDevices(params_invalid_param_name) -def test_no_data(requester): +def test_no_data(requester, err_404): params_no_data = {"deviceCode": "BPR-Folger-59", "dateTo": "1900-01-01"} - with pytest.raises(requests.HTTPError, match=r"404 Client Error"): + with pytest.raises(requests.HTTPError, match=err_404): requester.getDevices(params_no_data) diff --git a/tests/discover_locations/test_locations.py b/tests/discover_locations/test_locations.py index 2160042..af60aa0 100644 --- a/tests/discover_locations/test_locations.py +++ b/tests/discover_locations/test_locations.py @@ -2,40 +2,40 @@ import requests -def test_invalid_time_range_greater_start_time(requester): +def test_invalid_time_range_greater_start_time(requester, err_400): params_invalid_time_range_greater_start_time = { "locationCode": "FGPD", "dateFrom": "2020-01-01", "dateTo": "2019-01-01", } - with pytest.raises(requests.HTTPError, match=r"API Error 23"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getLocations(params_invalid_time_range_greater_start_time) -def test_invalid_time_range_future_start_time(requester): +def test_invalid_time_range_future_start_time(requester, err_400): params_invalid_time_range_future_start_time = { "locationCode": "FGPD", "dateFrom": "2050-01-01", } - with pytest.raises(requests.HTTPError, match=r"API Error 25"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getLocations(params_invalid_time_range_future_start_time) -def test_invalid_param_value(requester): +def test_invalid_param_value(requester, err_400): params_invalid_param_value = {"locationCode": "XYZ123"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getLocations(params_invalid_param_value) -def test_invalid_param_name(requester): +def test_invalid_param_name(requester, err_400): params_invalid_param_name = {"locationCodes": "FGPD"} - with pytest.raises(requests.HTTPError, match=r"API Error 129"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getLocations(params_invalid_param_name) -def test_no_data(requester): +def test_no_data(requester, err_404): params_no_data = {"locationCode": "FGPD", "dateTo": "1900-01-01"} - with pytest.raises(requests.HTTPError, match=r"404 Client Error"): + with pytest.raises(requests.HTTPError, match=err_404): requester.getLocations(params_no_data) diff --git a/tests/discover_locations/test_locations_tree.py b/tests/discover_locations/test_locations_tree.py index e0eb47a..388c06a 100644 --- a/tests/discover_locations/test_locations_tree.py +++ b/tests/discover_locations/test_locations_tree.py @@ -2,40 +2,40 @@ import requests -def test_invalid_time_range_greater_start_time(requester): +def test_invalid_time_range_greater_start_time(requester, err_400): params_invalid_time_range_greater_start_time = { "locationCode": "ARCT", "dateFrom": "2020-01-01", "dateTo": "2019-01-01", } - with pytest.raises(requests.HTTPError, match=r"API Error 23"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getLocationsTree(params_invalid_time_range_greater_start_time) -def test_invalid_time_range_future_start_time(requester): +def test_invalid_time_range_future_start_time(requester, err_400): params_invalid_time_range_future_start_time = { "locationCode": "ARCT", "dateFrom": "2050-01-01", } - with pytest.raises(requests.HTTPError, match=r"API Error 25"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getLocationsTree(params_invalid_time_range_future_start_time) -def test_invalid_param_value(requester): +def test_invalid_param_value(requester, err_400): params_invalid_param_value = {"locationCode": "XYZ123"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getLocationsTree(params_invalid_param_value) -def test_invalid_param(requester): +def test_invalid_param(requester, err_400): params_invalid_param_name = {"locationCodes": "ARCT"} - with pytest.raises(requests.HTTPError, match=r"API Error 129"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getLocationsTree(params_invalid_param_name) -def test_no_data(requester): +def test_no_data(requester, err_404): params_no_data = {"locationCode": "ARCT", "dateTo": "1900-01-01"} - with pytest.raises(requests.HTTPError, match=r"404 Client Error"): + with pytest.raises(requests.HTTPError, match=err_404): requester.getLocationsTree(params_no_data) diff --git a/tests/discover_properties/test_properties.py b/tests/discover_properties/test_properties.py index a72d88b..6703601 100644 --- a/tests/discover_properties/test_properties.py +++ b/tests/discover_properties/test_properties.py @@ -2,25 +2,25 @@ import requests -def test_invalid_param_value(requester): +def test_invalid_param_value(requester, err_400): params_invalid_param_value = {"propertyCode": "XYZ123"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getProperties(params_invalid_param_value) -def test_invalid_param_name(requester): +def test_invalid_param_name(requester, err_400): params_invalid_param_name = {"propertyCodes": "conductivity"} - with pytest.raises(requests.HTTPError, match=r"API Error 129"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getProperties(params_invalid_param_name) -def test_no_data(requester): +def test_no_data(requester, err_404): params_no_data = { "propertyCode": "conductivity", "locationCode": "SAAN", } - with pytest.raises(requests.HTTPError, match=r"404 Client Error"): + with pytest.raises(requests.HTTPError, match=err_404): requester.getProperties(params_no_data) diff --git a/tests/raw_data/test_rawdata_device.py b/tests/raw_data/test_rawdata_device.py index 2e13959..6fe6585 100644 --- a/tests/raw_data/test_rawdata_device.py +++ b/tests/raw_data/test_rawdata_device.py @@ -18,15 +18,15 @@ def params_multiple_pages(params): return params | {"rowLimit": 25} -def test_invalid_param_value(requester, params): +def test_invalid_param_value(requester, params, err_400): params_invalid_param_value = params | {"deviceCode": "XYZ123"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getRawdata(params_invalid_param_value) -def test_invalid_param_name(requester, params): +def test_invalid_param_name(requester, params, err_400): params_invalid_param_name = params | {"deviceCodes": "BPR-Folger-59"} - with pytest.raises(requests.HTTPError, match=r"API Error 129"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getRawdata(params_invalid_param_name) diff --git a/tests/raw_data/test_rawdata_location.py b/tests/raw_data/test_rawdata_location.py index e9af23f..9889c45 100644 --- a/tests/raw_data/test_rawdata_location.py +++ b/tests/raw_data/test_rawdata_location.py @@ -21,21 +21,21 @@ def params_multiple_pages(params): return params | {"rowLimit": 25} -def test_invalid_param_value(requester, params): +def test_invalid_param_value(requester, params, err_400): params_invalid_param_value = params | {"locationCode": "XYZ123"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getRawdata(params_invalid_param_value) -def test_invalid_param_name(requester, params): +def test_invalid_param_name(requester, params, err_400): params_invalid_param_name = params | {"locationCodes": "NCBC"} - with pytest.raises(requests.HTTPError, match=r"API Error 129"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getRawdata(params_invalid_param_name) -def test_no_data(requester, params): +def test_no_data(requester, params, err_400): params_no_data = params | {"dateFrom": "2000-01-01", "dateTo": "2000-01-02"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getRawdata(params_no_data) diff --git a/tests/scalar_data/test_scalardata_device.py b/tests/scalar_data/test_scalardata_device.py index ffbb63e..c85d46a 100644 --- a/tests/scalar_data/test_scalardata_device.py +++ b/tests/scalar_data/test_scalardata_device.py @@ -8,15 +8,15 @@ def params_multiple_pages(params_device): return params_device | {"rowLimit": 25} -def test_invalid_param_value(requester, params_device): +def test_invalid_param_value(requester, params_device, err_400): params_invalid_param_value = params_device | {"deviceCode": "XYZ123"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getScalardata(params_invalid_param_value) -def test_invalid_param_name(requester, params_device): +def test_invalid_param_name(requester, params_device, err_400): params_invalid_param_name = params_device | {"deviceCodes": "BPR-Folger-59"} - with pytest.raises(requests.HTTPError, match=r"API Error 129"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getScalardata(params_invalid_param_name) diff --git a/tests/scalar_data/test_scalardata_location.py b/tests/scalar_data/test_scalardata_location.py index 19ed397..bb38752 100644 --- a/tests/scalar_data/test_scalardata_location.py +++ b/tests/scalar_data/test_scalardata_location.py @@ -8,24 +8,24 @@ def params_multiple_pages(params_location): return params_location | {"rowLimit": 25} -def test_invalid_param_value(requester, params_location): +def test_invalid_param_value(requester, params_location, err_400): params_invalid_param_value = params_location | {"locationCode": "XYZ123"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getScalardata(params_invalid_param_value) -def test_invalid_param_name(requester, params_location): +def test_invalid_param_name(requester, params_location, err_400): params_invalid_param_name = params_location | {"locationCodes": "NCBC"} - with pytest.raises(requests.HTTPError, match=r"API Error 129"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getScalardata(params_invalid_param_name) -def test_no_data(requester, params_location): +def test_no_data(requester, params_location, err_400): params_no_data = params_location | { "dateFrom": "2000-01-01", "dateTo": "2000-01-02", } - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getScalardata(params_no_data)