From 6db8622916500d1db8bc5b954291e60647b96259 Mon Sep 17 00:00:00 2001 From: Ian Black Date: Mon, 10 Nov 2025 13:19:43 -0800 Subject: [PATCH 1/4] Add functionality for more descriptive console messages and returning requests objects. --- scratch.ipynb | 2121 ++++++++++++++++++++++++++++++ src/onc/modules/_Messages.py | 86 ++ src/onc/modules/_MultiPage.py | 83 +- src/onc/modules/_OncArchive.py | 5 +- src/onc/modules/_OncDelivery.py | 4 +- src/onc/modules/_OncDiscovery.py | 5 +- src/onc/modules/_OncRealTime.py | 6 +- src/onc/modules/_OncService.py | 107 +- src/onc/modules/_util.py | 1 + src/onc/onc.py | 34 +- 10 files changed, 2351 insertions(+), 101 deletions(-) create mode 100644 scratch.ipynb create mode 100644 src/onc/modules/_Messages.py diff --git a/scratch.ipynb b/scratch.ipynb new file mode 100644 index 0000000..06646c4 --- /dev/null +++ b/scratch.ipynb @@ -0,0 +1,2121 @@ +{ + "cells": [ + { + "cell_type": "code", + "id": "initial_id", + "metadata": { + "collapsed": true, + "ExecuteTime": { + "end_time": "2025-11-10T21:17:04.711252Z", + "start_time": "2025-11-10T21:17:04.581618Z" + } + }, + "source": [ + "from netrc import netrc\n", + "from onc import ONC\n", + "\n", + "_, __, token = netrc().authenticators('data.oceannetworks.ca')" + ], + "outputs": [], + "execution_count": 1 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "## Initialization", + "id": "ef917e2e8133e385" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-11-10T21:17:04.722824Z", + "start_time": "2025-11-10T21:17:04.717615Z" + } + }, + "cell_type": "code", + "source": [ + "message_level = 'DEBUG' # Prints all messages to console.\n", + "redact_token = True # Removes a users token from console messages if set to True.\n", + "raise_http_errors = False # Does not raise HTTPError and instead returns the full requests.Response object.\n", + "\n", + "onc = ONC(token = token, verbosity = message_level, redact_token=redact_token, raise_http_errors=raise_http_errors)" + ], + "id": "f762aa13502fee66", + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-11-10T21:17:04.720Z | onc-client | DEBUG | Initialized ONC module.\n" + ] + } + ], + "execution_count": 2 + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-11-10T21:00:57.054214Z", + "start_time": "2025-11-10T21:00:57.049825Z" + } + }, + "cell_type": "markdown", + "source": "## OK Discovery Example", + "id": "9b3a0033f2f05f6b" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-11-10T21:17:05.247467Z", + "start_time": "2025-11-10T21:17:04.729847Z" + } + }, + "cell_type": "code", + "source": [ + "locs = onc.getLocationsTree({'locationCode': 'BCF'})\n", + "locs" + ], + "id": "5e6ed0b663a30da0", + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-11-10T21:17:05.232Z | onc-service | INFO | Requested: https://data.oceannetworks.ca/api/locations?locationCode=BCF&method=getTree&token=REDACTED\n", + "2025-11-10T21:17:05.232Z | onc-service | DEBUG | Response received in 0.5 seconds.\n", + "2025-11-10T21:17:05.232Z | onc-service | INFO | HTTP Response: OK (200)\n" + ] + }, + { + "data": { + "text/plain": [ + "[{'locationName': 'British Columbia Ferries',\n", + " 'children': [{'locationName': 'Campbell River - Quathiaski Cove Ferry Route',\n", + " 'children': None,\n", + " 'description': 'This ferry route goes between Campbell River, Vancouver Island, BC, and Quadra Island (Quathiaski Cove), BC.',\n", + " 'hasDeviceData': True,\n", + " 'locationCode': 'CRQC',\n", + " 'hasPropertyData': True},\n", + " {'locationName': 'Horseshoe Bay - Departure Bay Ferry Route',\n", + " 'children': [{'locationName': 'GNSS V104S GPS',\n", + " 'children': None,\n", + " 'description': None,\n", + " 'hasDeviceData': True,\n", + " 'locationCode': 'HBDB.N1',\n", + " 'hasPropertyData': True},\n", + " {'locationName': 'Thrane and Thrane SAILOR',\n", + " 'children': None,\n", + " 'description': None,\n", + " 'hasDeviceData': True,\n", + " 'locationCode': 'HBDB.N2',\n", + " 'hasPropertyData': False}],\n", + " 'description': 'This ferry route goes between Horseshoe Bay terminal in West Vancouver, BC and the Departure Bay terminal, Nanaimo BC.',\n", + " 'hasDeviceData': True,\n", + " 'locationCode': 'HBDB',\n", + " 'hasPropertyData': True},\n", + " {'locationName': 'Nanaimo Harbour - Descanso Bay Ferry Route',\n", + " 'children': None,\n", + " 'description': 'This ferry route goes between Nanaimo Harbour terminal in Nanaimo, BC and the Descanso Bay terminal on Gabriola Island, BC.',\n", + " 'hasDeviceData': True,\n", + " 'locationCode': 'NHDB',\n", + " 'hasPropertyData': True},\n", + " {'locationName': 'Port McNeill - Alert Bay - Sointula Ferry Route',\n", + " 'children': None,\n", + " 'description': 'This ferry route goes between Cormorant Island (Alert Bay), BC, Port McNeill, BC and Malcolm Island (Sointula), BC.',\n", + " 'hasDeviceData': True,\n", + " 'locationCode': 'PMABS',\n", + " 'hasPropertyData': True},\n", + " {'locationName': 'Tsawwassen - Duke Point Ferry Route',\n", + " 'children': [{'locationName': 'GNSS MV104S GPS',\n", + " 'children': None,\n", + " 'description': '',\n", + " 'hasDeviceData': True,\n", + " 'locationCode': 'TWDP.N1',\n", + " 'hasPropertyData': True},\n", + " {'locationName': 'Thrane and Thrane SAILOR',\n", + " 'children': None,\n", + " 'description': None,\n", + " 'hasDeviceData': True,\n", + " 'locationCode': 'TWDP.N2',\n", + " 'hasPropertyData': False}],\n", + " 'description': 'This ferry route goes between Tsawwassen terminal in Vancouver, BC and the Duke Point terminal, Nanaimo BC.',\n", + " 'hasDeviceData': True,\n", + " 'locationCode': 'TWDP',\n", + " 'hasPropertyData': True},\n", + " {'locationName': 'Tsawwassen - Swartz Bay Ferry Route',\n", + " 'children': [{'locationName': 'Fluorometer Secondary',\n", + " 'children': None,\n", + " 'description': '',\n", + " 'hasDeviceData': True,\n", + " 'locationCode': 'TWSB.F1',\n", + " 'hasPropertyData': False},\n", + " {'locationName': 'GNSS V104S GPS',\n", + " 'children': None,\n", + " 'description': None,\n", + " 'hasDeviceData': True,\n", + " 'locationCode': 'TWSB.N1',\n", + " 'hasPropertyData': False},\n", + " {'locationName': 'Thrane and Thrane SAILOR',\n", + " 'children': None,\n", + " 'description': None,\n", + " 'hasDeviceData': True,\n", + " 'locationCode': 'TWSB.N2',\n", + " 'hasPropertyData': False}],\n", + " 'description': 'This ferry route goes between Tsawwassen terminal in Vancouver, BC and the Swartz Bay terminal, Victoria BC.',\n", + " 'hasDeviceData': True,\n", + " 'locationCode': 'TWSB',\n", + " 'hasPropertyData': True}],\n", + " 'description': ' A few ferry vessels in the Salish Sea have been equipped with scientific instruments. Data is collected from routes between Vancouver Island and the mainland of British Columbia. Data is transmitted to Ocean Networks Canada shore stations via a wireless connection.',\n", + " 'hasDeviceData': False,\n", + " 'locationCode': 'BCF',\n", + " 'hasPropertyData': False}]" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 3 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "## Not Found Discovery Example", + "id": "f983589d8201f9d3" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-11-10T21:17:06.001885Z", + "start_time": "2025-11-10T21:17:05.290709Z" + } + }, + "cell_type": "code", + "source": [ + "locs = onc.getLocations({'locationCode': 'BCF',\n", + " 'dateFrom': '2000-01-01T00:00:00.000Z',\n", + " 'dateTo': '2000-01-01T23:59:59.999Z'})\n", + "locs.status_code # Output is a requests.Response object instead of a raised HTTPError" + ], + "id": "5e9b97e55e7b9419", + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-11-10T21:17:05.992Z | onc-service | INFO | Requested: https://data.oceannetworks.ca/api/locations?locationCode=BCF&dateFrom=2000-01-01T00%3A00%3A00.000Z&dateTo=2000-01-01T23%3A59%3A59.999Z&method=get&token=REDACTED\n", + "2025-11-10T21:17:05.992Z | onc-service | DEBUG | Response received in 0.695 seconds.\n", + "2025-11-10T21:17:05.992Z | onc-service | ERROR | HTTP Response: Not Found (404)\n", + "2025-11-10T21:17:05.992Z | onc-service | ERROR | No metadata/data found for the given query parameters. Check if parameter input is valid.\n" + ] + }, + { + "data": { + "text/plain": [ + "404" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 4 + }, + { + "metadata": {}, + "cell_type": "markdown", + "source": "## OK Data Example", + "id": "4c7c24a4da781d60" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-11-10T21:18:00.702793Z", + "start_time": "2025-11-10T21:17:54.596415Z" + } + }, + "cell_type": "code", + "source": [ + "data = onc.getScalardata({'locationCode':'BACVP',\n", + " 'deviceCategoryCode':'CTD',\n", + " 'dateFrom': '2009-08-20T00:00:00.000Z',\n", + " 'dateTo': '2009-08-29T23:59:59.999Z',\n", + " 'rowLimit': 10}, allPages = True)\n", + "data['sensorData']" + ], + "id": "16e99050e0ab4c0a", + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-11-10T21:17:55.210Z | onc-service | INFO | Requested: https://data.oceannetworks.ca/api/scalardata?locationCode=BACVP&deviceCategoryCode=CTD&dateFrom=2009-08-20T00%3A00%3A00.000Z&dateTo=2009-08-29T23%3A59%3A59.999Z&rowLimit=10&method=getByLocation&token=REDACTED\n", + "2025-11-10T21:17:55.213Z | onc-service | DEBUG | Response received in 0.608 seconds.\n", + "2025-11-10T21:17:55.213Z | onc-service | INFO | HTTP Response: OK (200)\n", + "2025-11-10T21:17:55.217Z | onc-multi | INFO | The requested data quantity is greater than the supplied row limit and will be downloaded over multiple requests.\n", + "2025-11-10T21:17:55.219Z | onc-multi | DEBUG | Download time for page 1: 0.61 seconds\n", + "2025-11-10T21:17:55.222Z | onc-multi | INFO | Est. number of pages remaining for download: 114332\n", + "2025-11-10T21:17:55.222Z | onc-multi | INFO | Est. number of seconds to download remaining data: 19 hours\n", + "2025-11-10T21:17:55.223Z | onc-multi | DEBUG | Submitting request for page 2 (10 samples)...\n", + "2025-11-10T21:17:55.852Z | onc-service | INFO | Requested: https://data.oceannetworks.ca/api/scalardata?method=getByLocation&dateTo=2009-08-29T23%3A59%3A59.999Z&rowLimit=10&locationCode=BACVP&deviceCategoryCode=CTD&dateFrom=2009-08-27T00%3A53%3A32.289Z&token=REDACTED\n", + "2025-11-10T21:17:55.852Z | onc-service | DEBUG | Response received in 0.626 seconds.\n", + "2025-11-10T21:17:55.852Z | onc-service | INFO | HTTP Response: OK (200)\n", + "2025-11-10T21:17:55.858Z | onc-multi | DEBUG | Submitting request for page 3 (20 samples)...\n", + "2025-11-10T21:17:56.461Z | onc-service | INFO | Requested: https://data.oceannetworks.ca/api/scalardata?method=getByLocation&dateTo=2009-08-29T23%3A59%3A59.999Z&rowLimit=10&locationCode=BACVP&deviceCategoryCode=CTD&dateFrom=2009-08-27T00%3A54%3A33.083Z&token=REDACTED\n", + "2025-11-10T21:17:56.461Z | onc-service | DEBUG | Response received in 0.607 seconds.\n", + "2025-11-10T21:17:56.471Z | onc-service | INFO | HTTP Response: OK (200)\n", + "2025-11-10T21:17:56.471Z | onc-multi | DEBUG | Submitting request for page 4 (30 samples)...\n", + "2025-11-10T21:17:57.100Z | onc-service | INFO | Requested: https://data.oceannetworks.ca/api/scalardata?method=getByLocation&dateTo=2009-08-29T23%3A59%3A59.999Z&rowLimit=10&locationCode=BACVP&deviceCategoryCode=CTD&dateFrom=2009-08-27T00%3A55%3A33.075Z&token=REDACTED\n", + "2025-11-10T21:17:57.100Z | onc-service | DEBUG | Response received in 0.625 seconds.\n", + "2025-11-10T21:17:57.100Z | onc-service | INFO | HTTP Response: OK (200)\n", + "2025-11-10T21:17:57.107Z | onc-multi | DEBUG | Submitting request for page 5 (40 samples)...\n", + "2025-11-10T21:17:57.697Z | onc-service | INFO | Requested: https://data.oceannetworks.ca/api/scalardata?method=getByLocation&dateTo=2009-08-29T23%3A59%3A59.999Z&rowLimit=10&locationCode=BACVP&deviceCategoryCode=CTD&dateFrom=2009-08-27T00%3A56%3A32.975Z&token=REDACTED\n", + "2025-11-10T21:17:57.707Z | onc-service | DEBUG | Response received in 0.593 seconds.\n", + "2025-11-10T21:17:57.707Z | onc-service | INFO | HTTP Response: OK (200)\n", + "2025-11-10T21:17:57.707Z | onc-multi | DEBUG | Submitting request for page 6 (50 samples)...\n", + "2025-11-10T21:17:58.271Z | onc-service | INFO | Requested: https://data.oceannetworks.ca/api/scalardata?method=getByLocation&dateTo=2009-08-29T23%3A59%3A59.999Z&rowLimit=10&locationCode=BACVP&deviceCategoryCode=CTD&dateFrom=2009-08-27T00%3A57%3A32.881Z&token=REDACTED\n", + "2025-11-10T21:17:58.271Z | onc-service | DEBUG | Response received in 0.556 seconds.\n", + "2025-11-10T21:17:58.271Z | onc-service | INFO | HTTP Response: OK (200)\n", + "2025-11-10T21:17:58.271Z | onc-multi | DEBUG | Submitting request for page 7 (60 samples)...\n", + "2025-11-10T21:17:58.833Z | onc-service | INFO | Requested: https://data.oceannetworks.ca/api/scalardata?method=getByLocation&dateTo=2009-08-29T23%3A59%3A59.999Z&rowLimit=10&locationCode=BACVP&deviceCategoryCode=CTD&dateFrom=2009-08-27T00%3A58%3A32.891Z&token=REDACTED\n", + "2025-11-10T21:17:58.833Z | onc-service | DEBUG | Response received in 0.555 seconds.\n", + "2025-11-10T21:17:58.833Z | onc-service | INFO | HTTP Response: OK (200)\n", + "2025-11-10T21:17:58.833Z | onc-multi | DEBUG | Submitting request for page 8 (70 samples)...\n", + "2025-11-10T21:17:59.440Z | onc-service | INFO | Requested: https://data.oceannetworks.ca/api/scalardata?method=getByLocation&dateTo=2009-08-29T23%3A59%3A59.999Z&rowLimit=10&locationCode=BACVP&deviceCategoryCode=CTD&dateFrom=2009-08-27T00%3A59%3A32.886Z&token=REDACTED\n", + "2025-11-10T21:17:59.440Z | onc-service | DEBUG | Response received in 0.597 seconds.\n", + "2025-11-10T21:17:59.440Z | onc-service | INFO | HTTP Response: OK (200)\n", + "2025-11-10T21:17:59.440Z | onc-multi | DEBUG | Submitting request for page 9 (80 samples)...\n", + "2025-11-10T21:18:00.066Z | onc-service | INFO | Requested: https://data.oceannetworks.ca/api/scalardata?method=getByLocation&dateTo=2009-08-29T23%3A59%3A59.999Z&rowLimit=10&locationCode=BACVP&deviceCategoryCode=CTD&dateFrom=2009-08-27T01%3A00%3A32.910Z&token=REDACTED\n", + "2025-11-10T21:18:00.066Z | onc-service | DEBUG | Response received in 0.619 seconds.\n", + "2025-11-10T21:18:00.066Z | onc-service | INFO | HTTP Response: OK (200)\n", + "2025-11-10T21:18:00.074Z | onc-multi | DEBUG | Submitting request for page 10 (90 samples)...\n", + "2025-11-10T21:18:00.676Z | onc-service | INFO | Requested: https://data.oceannetworks.ca/api/scalardata?method=getByLocation&dateTo=2009-08-29T23%3A59%3A59.999Z&rowLimit=10&locationCode=BACVP&deviceCategoryCode=CTD&dateFrom=2009-08-27T01%3A01%3A32.909Z&token=REDACTED\n", + "2025-11-10T21:18:00.676Z | onc-service | DEBUG | Response received in 0.6 seconds.\n", + "2025-11-10T21:18:00.676Z | onc-service | INFO | HTTP Response: OK (200)\n", + "2025-11-10T21:18:00.687Z | onc-multi | INFO | Downloaded 94 total samples in 6 seconds.\n" + ] + }, + { + "data": { + "text/plain": [ + "[{'actualSamples': 3,\n", + " 'data': {'qaqcFlags': [0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0],\n", + " 'sampleTimes': ['2009-08-27T00:53:29.800Z',\n", + " '2009-08-27T00:53:30.043Z',\n", + " '2009-08-27T00:53:30.289Z',\n", + " '2009-08-27T00:53:30.539Z',\n", + " '2009-08-27T00:53:30.789Z',\n", + " '2009-08-27T00:53:31.039Z',\n", + " '2009-08-27T00:53:31.289Z',\n", + " '2009-08-27T00:53:31.539Z',\n", + " '2009-08-27T00:53:31.789Z',\n", + " '2009-08-27T00:53:32.039Z',\n", + " '2009-08-27T00:54:30.585Z',\n", + " '2009-08-27T00:54:30.834Z',\n", + " '2009-08-27T00:54:31.084Z',\n", + " '2009-08-27T00:54:31.333Z',\n", + " '2009-08-27T00:54:31.583Z',\n", + " '2009-08-27T00:54:31.833Z',\n", + " '2009-08-27T00:54:32.083Z',\n", + " '2009-08-27T00:54:32.333Z',\n", + " '2009-08-27T00:54:32.583Z',\n", + " '2009-08-27T00:54:32.833Z',\n", + " '2009-08-27T00:55:30.578Z',\n", + " '2009-08-27T00:55:30.827Z',\n", + " '2009-08-27T00:55:31.077Z',\n", + " '2009-08-27T00:55:31.325Z',\n", + " '2009-08-27T00:55:31.575Z',\n", + " '2009-08-27T00:55:31.825Z',\n", + " '2009-08-27T00:55:32.075Z',\n", + " '2009-08-27T00:55:32.325Z',\n", + " '2009-08-27T00:55:32.575Z',\n", + " '2009-08-27T00:55:32.825Z',\n", + " '2009-08-27T00:56:30.597Z',\n", + " '2009-08-27T00:56:30.847Z',\n", + " '2009-08-27T00:56:31.033Z',\n", + " '2009-08-27T00:56:31.225Z',\n", + " '2009-08-27T00:56:31.475Z',\n", + " '2009-08-27T00:56:31.725Z',\n", + " '2009-08-27T00:56:31.975Z',\n", + " '2009-08-27T00:56:32.225Z',\n", + " '2009-08-27T00:56:32.475Z',\n", + " '2009-08-27T00:56:32.725Z',\n", + " '2009-08-27T00:57:30.380Z',\n", + " '2009-08-27T00:57:30.631Z',\n", + " '2009-08-27T00:57:30.882Z',\n", + " '2009-08-27T00:57:31.131Z',\n", + " '2009-08-27T00:57:31.381Z',\n", + " '2009-08-27T00:57:31.631Z',\n", + " '2009-08-27T00:57:31.881Z',\n", + " '2009-08-27T00:57:32.131Z',\n", + " '2009-08-27T00:57:32.381Z',\n", + " '2009-08-27T00:57:32.631Z',\n", + " '2009-08-27T00:58:30.387Z',\n", + " '2009-08-27T00:58:30.638Z',\n", + " '2009-08-27T00:58:30.889Z',\n", + " '2009-08-27T00:58:31.141Z',\n", + " '2009-08-27T00:58:31.391Z',\n", + " '2009-08-27T00:58:31.641Z',\n", + " '2009-08-27T00:58:31.891Z',\n", + " '2009-08-27T00:58:32.141Z',\n", + " '2009-08-27T00:58:32.391Z',\n", + " '2009-08-27T00:58:32.641Z',\n", + " '2009-08-27T00:59:30.389Z',\n", + " '2009-08-27T00:59:30.639Z',\n", + " '2009-08-27T00:59:30.888Z',\n", + " '2009-08-27T00:59:31.136Z',\n", + " '2009-08-27T00:59:31.386Z',\n", + " '2009-08-27T00:59:31.636Z',\n", + " '2009-08-27T00:59:31.886Z',\n", + " '2009-08-27T00:59:32.136Z',\n", + " '2009-08-27T00:59:32.386Z',\n", + " '2009-08-27T00:59:32.636Z',\n", + " '2009-08-27T01:00:30.407Z',\n", + " '2009-08-27T01:00:30.657Z',\n", + " '2009-08-27T01:00:30.907Z',\n", + " '2009-08-27T01:00:31.160Z',\n", + " '2009-08-27T01:00:31.410Z',\n", + " '2009-08-27T01:00:31.660Z',\n", + " '2009-08-27T01:00:31.910Z',\n", + " '2009-08-27T01:00:32.160Z',\n", + " '2009-08-27T01:00:32.410Z',\n", + " '2009-08-27T01:00:32.660Z',\n", + " '2009-08-27T01:01:30.408Z',\n", + " '2009-08-27T01:01:30.657Z',\n", + " '2009-08-27T01:01:30.909Z',\n", + " '2009-08-27T01:01:31.159Z',\n", + " '2009-08-27T01:01:31.409Z',\n", + " '2009-08-27T01:01:31.659Z',\n", + " '2009-08-27T01:01:31.909Z',\n", + " '2009-08-27T01:01:32.159Z',\n", + " '2009-08-27T01:01:32.409Z',\n", + " '2009-08-27T01:01:32.659Z',\n", + " '2009-08-27T01:02:30.424Z',\n", + " '2009-08-27T01:02:30.674Z',\n", + " '2009-08-27T01:02:30.923Z',\n", + " '2009-08-27T01:02:31.173Z'],\n", + " 'values': [3.35513,\n", + " 3.35514,\n", + " 3.35505,\n", + " 3.35503,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 3.35511,\n", + " 3.35509,\n", + " 3.35506,\n", + " 3.35504,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 3.35504,\n", + " 3.35507,\n", + " 3.35511,\n", + " 3.35513,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 3.35501,\n", + " 3.35496,\n", + " 3.35495,\n", + " 3.35499,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 3.35499,\n", + " 3.35498,\n", + " 3.35496,\n", + " 3.35485,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 3.35488,\n", + " 3.35482,\n", + " 3.35477,\n", + " 3.35476,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 3.35476,\n", + " 3.35473,\n", + " 3.35464,\n", + " 3.3546,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 3.35469,\n", + " 3.35464,\n", + " 3.35463,\n", + " 3.35456,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 3.35466,\n", + " 3.35465,\n", + " 3.35462,\n", + " 3.35463,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 3.35462,\n", + " 3.35461,\n", + " 3.3546,\n", + " 3.35458]},\n", + " 'outputFormat': 'array',\n", + " 'propertyCode': 'conductivity',\n", + " 'sensorCategoryCode': 'conductivity',\n", + " 'sensorCode': 'cond',\n", + " 'sensorName': 'Conductivity',\n", + " 'unitOfMeasure': 'S/m'},\n", + " {'actualSamples': 3,\n", + " 'data': {'qaqcFlags': [0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0],\n", + " 'sampleTimes': ['2009-08-27T00:53:29.800Z',\n", + " '2009-08-27T00:53:30.043Z',\n", + " '2009-08-27T00:53:30.289Z',\n", + " '2009-08-27T00:53:30.539Z',\n", + " '2009-08-27T00:53:30.789Z',\n", + " '2009-08-27T00:53:31.039Z',\n", + " '2009-08-27T00:53:31.289Z',\n", + " '2009-08-27T00:53:31.539Z',\n", + " '2009-08-27T00:53:31.789Z',\n", + " '2009-08-27T00:53:32.039Z',\n", + " '2009-08-27T00:54:30.585Z',\n", + " '2009-08-27T00:54:30.834Z',\n", + " '2009-08-27T00:54:31.084Z',\n", + " '2009-08-27T00:54:31.333Z',\n", + " '2009-08-27T00:54:31.583Z',\n", + " '2009-08-27T00:54:31.833Z',\n", + " '2009-08-27T00:54:32.083Z',\n", + " '2009-08-27T00:54:32.333Z',\n", + " '2009-08-27T00:54:32.583Z',\n", + " '2009-08-27T00:54:32.833Z',\n", + " '2009-08-27T00:55:30.578Z',\n", + " '2009-08-27T00:55:30.827Z',\n", + " '2009-08-27T00:55:31.077Z',\n", + " '2009-08-27T00:55:31.325Z',\n", + " '2009-08-27T00:55:31.575Z',\n", + " '2009-08-27T00:55:31.825Z',\n", + " '2009-08-27T00:55:32.075Z',\n", + " '2009-08-27T00:55:32.325Z',\n", + " '2009-08-27T00:55:32.575Z',\n", + " '2009-08-27T00:55:32.825Z',\n", + " '2009-08-27T00:56:30.597Z',\n", + " '2009-08-27T00:56:30.847Z',\n", + " '2009-08-27T00:56:31.033Z',\n", + " '2009-08-27T00:56:31.225Z',\n", + " '2009-08-27T00:56:31.475Z',\n", + " '2009-08-27T00:56:31.725Z',\n", + " '2009-08-27T00:56:31.975Z',\n", + " '2009-08-27T00:56:32.225Z',\n", + " '2009-08-27T00:56:32.475Z',\n", + " '2009-08-27T00:56:32.725Z',\n", + " '2009-08-27T00:57:30.380Z',\n", + " '2009-08-27T00:57:30.631Z',\n", + " '2009-08-27T00:57:30.882Z',\n", + " '2009-08-27T00:57:31.131Z',\n", + " '2009-08-27T00:57:31.381Z',\n", + " '2009-08-27T00:57:31.631Z',\n", + " '2009-08-27T00:57:31.881Z',\n", + " '2009-08-27T00:57:32.131Z',\n", + " '2009-08-27T00:57:32.381Z',\n", + " '2009-08-27T00:57:32.631Z',\n", + " '2009-08-27T00:58:30.387Z',\n", + " '2009-08-27T00:58:30.638Z',\n", + " '2009-08-27T00:58:30.889Z',\n", + " '2009-08-27T00:58:31.141Z',\n", + " '2009-08-27T00:58:31.391Z',\n", + " '2009-08-27T00:58:31.641Z',\n", + " '2009-08-27T00:58:31.891Z',\n", + " '2009-08-27T00:58:32.141Z',\n", + " '2009-08-27T00:58:32.391Z',\n", + " '2009-08-27T00:58:32.641Z',\n", + " '2009-08-27T00:59:30.389Z',\n", + " '2009-08-27T00:59:30.639Z',\n", + " '2009-08-27T00:59:30.888Z',\n", + " '2009-08-27T00:59:31.136Z',\n", + " '2009-08-27T00:59:31.386Z',\n", + " '2009-08-27T00:59:31.636Z',\n", + " '2009-08-27T00:59:31.886Z',\n", + " '2009-08-27T00:59:32.136Z',\n", + " '2009-08-27T00:59:32.386Z',\n", + " '2009-08-27T00:59:32.636Z',\n", + " '2009-08-27T01:00:30.407Z',\n", + " '2009-08-27T01:00:30.657Z',\n", + " '2009-08-27T01:00:30.907Z',\n", + " '2009-08-27T01:00:31.160Z',\n", + " '2009-08-27T01:00:31.410Z',\n", + " '2009-08-27T01:00:31.660Z',\n", + " '2009-08-27T01:00:31.910Z',\n", + " '2009-08-27T01:00:32.160Z',\n", + " '2009-08-27T01:00:32.410Z',\n", + " '2009-08-27T01:00:32.660Z',\n", + " '2009-08-27T01:01:30.408Z',\n", + " '2009-08-27T01:01:30.657Z',\n", + " '2009-08-27T01:01:30.909Z',\n", + " '2009-08-27T01:01:31.159Z',\n", + " '2009-08-27T01:01:31.409Z',\n", + " '2009-08-27T01:01:31.659Z',\n", + " '2009-08-27T01:01:31.909Z',\n", + " '2009-08-27T01:01:32.159Z',\n", + " '2009-08-27T01:01:32.409Z',\n", + " '2009-08-27T01:01:32.659Z',\n", + " '2009-08-27T01:02:30.424Z',\n", + " '2009-08-27T01:02:30.674Z',\n", + " '2009-08-27T01:02:30.923Z',\n", + " '2009-08-27T01:02:31.173Z'],\n", + " 'values': [393.13,\n", + " 397.98,\n", + " 388.16,\n", + " 387.96,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 457.53,\n", + " 426.99,\n", + " 422.17,\n", + " 403.53,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 342.4,\n", + " 384.93,\n", + " 633.38,\n", + " 701.15,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 305.69,\n", + " 186.96,\n", + " 205.94,\n", + " 268.58,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 326.42,\n", + " 247.48,\n", + " 231.35,\n", + " 214.03,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 316.57,\n", + " 265.31,\n", + " 258.51,\n", + " 255.77,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 317.38,\n", + " 272.14,\n", + " 269.27,\n", + " 266.25,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 338.83,\n", + " 277.15,\n", + " 273.76,\n", + " 271.97,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 339.87,\n", + " 279.94,\n", + " 277.44,\n", + " 274.57,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 340.92,\n", + " 281.16,\n", + " 279.27,\n", + " 277.16]},\n", + " 'outputFormat': 'array',\n", + " 'propertyCode': 'depth',\n", + " 'sensorCategoryCode': 'depth',\n", + " 'sensorCode': 'depth',\n", + " 'sensorName': 'Depth',\n", + " 'unitOfMeasure': 'm'},\n", + " {'actualSamples': 3,\n", + " 'data': {'qaqcFlags': [0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0],\n", + " 'sampleTimes': ['2009-08-27T00:53:29.800Z',\n", + " '2009-08-27T00:53:30.043Z',\n", + " '2009-08-27T00:53:30.289Z',\n", + " '2009-08-27T00:53:30.539Z',\n", + " '2009-08-27T00:53:30.789Z',\n", + " '2009-08-27T00:53:31.039Z',\n", + " '2009-08-27T00:53:31.289Z',\n", + " '2009-08-27T00:53:31.539Z',\n", + " '2009-08-27T00:53:31.789Z',\n", + " '2009-08-27T00:53:32.039Z',\n", + " '2009-08-27T00:54:30.585Z',\n", + " '2009-08-27T00:54:30.834Z',\n", + " '2009-08-27T00:54:31.084Z',\n", + " '2009-08-27T00:54:31.333Z',\n", + " '2009-08-27T00:54:31.583Z',\n", + " '2009-08-27T00:54:31.833Z',\n", + " '2009-08-27T00:54:32.083Z',\n", + " '2009-08-27T00:54:32.333Z',\n", + " '2009-08-27T00:54:32.583Z',\n", + " '2009-08-27T00:54:32.833Z',\n", + " '2009-08-27T00:55:30.578Z',\n", + " '2009-08-27T00:55:30.827Z',\n", + " '2009-08-27T00:55:31.077Z',\n", + " '2009-08-27T00:55:31.325Z',\n", + " '2009-08-27T00:55:31.575Z',\n", + " '2009-08-27T00:55:31.825Z',\n", + " '2009-08-27T00:55:32.075Z',\n", + " '2009-08-27T00:55:32.325Z',\n", + " '2009-08-27T00:55:32.575Z',\n", + " '2009-08-27T00:55:32.825Z',\n", + " '2009-08-27T00:56:30.597Z',\n", + " '2009-08-27T00:56:30.847Z',\n", + " '2009-08-27T00:56:31.033Z',\n", + " '2009-08-27T00:56:31.225Z',\n", + " '2009-08-27T00:56:31.475Z',\n", + " '2009-08-27T00:56:31.725Z',\n", + " '2009-08-27T00:56:31.975Z',\n", + " '2009-08-27T00:56:32.225Z',\n", + " '2009-08-27T00:56:32.475Z',\n", + " '2009-08-27T00:56:32.725Z',\n", + " '2009-08-27T00:57:30.380Z',\n", + " '2009-08-27T00:57:30.631Z',\n", + " '2009-08-27T00:57:30.882Z',\n", + " '2009-08-27T00:57:31.131Z',\n", + " '2009-08-27T00:57:31.381Z',\n", + " '2009-08-27T00:57:31.631Z',\n", + " '2009-08-27T00:57:31.881Z',\n", + " '2009-08-27T00:57:32.131Z',\n", + " '2009-08-27T00:57:32.381Z',\n", + " '2009-08-27T00:57:32.631Z',\n", + " '2009-08-27T00:58:30.387Z',\n", + " '2009-08-27T00:58:30.638Z',\n", + " '2009-08-27T00:58:30.889Z',\n", + " '2009-08-27T00:58:31.141Z',\n", + " '2009-08-27T00:58:31.391Z',\n", + " '2009-08-27T00:58:31.641Z',\n", + " '2009-08-27T00:58:31.891Z',\n", + " '2009-08-27T00:58:32.141Z',\n", + " '2009-08-27T00:58:32.391Z',\n", + " '2009-08-27T00:58:32.641Z',\n", + " '2009-08-27T00:59:30.389Z',\n", + " '2009-08-27T00:59:30.639Z',\n", + " '2009-08-27T00:59:30.888Z',\n", + " '2009-08-27T00:59:31.136Z',\n", + " '2009-08-27T00:59:31.386Z',\n", + " '2009-08-27T00:59:31.636Z',\n", + " '2009-08-27T00:59:31.886Z',\n", + " '2009-08-27T00:59:32.136Z',\n", + " '2009-08-27T00:59:32.386Z',\n", + " '2009-08-27T00:59:32.636Z',\n", + " '2009-08-27T01:00:30.407Z',\n", + " '2009-08-27T01:00:30.657Z',\n", + " '2009-08-27T01:00:30.907Z',\n", + " '2009-08-27T01:00:31.160Z',\n", + " '2009-08-27T01:00:31.410Z',\n", + " '2009-08-27T01:00:31.660Z',\n", + " '2009-08-27T01:00:31.910Z',\n", + " '2009-08-27T01:00:32.160Z',\n", + " '2009-08-27T01:00:32.410Z',\n", + " '2009-08-27T01:00:32.660Z',\n", + " '2009-08-27T01:01:30.408Z',\n", + " '2009-08-27T01:01:30.657Z',\n", + " '2009-08-27T01:01:30.909Z',\n", + " '2009-08-27T01:01:31.159Z',\n", + " '2009-08-27T01:01:31.409Z',\n", + " '2009-08-27T01:01:31.659Z',\n", + " '2009-08-27T01:01:31.909Z',\n", + " '2009-08-27T01:01:32.159Z',\n", + " '2009-08-27T01:01:32.409Z',\n", + " '2009-08-27T01:01:32.659Z',\n", + " '2009-08-27T01:02:30.424Z',\n", + " '2009-08-27T01:02:30.674Z',\n", + " '2009-08-27T01:02:30.923Z',\n", + " '2009-08-27T01:02:31.173Z'],\n", + " 'values': [396.856,\n", + " 401.752,\n", + " 391.834,\n", + " 391.625,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 461.933,\n", + " 431.072,\n", + " 426.201,\n", + " 407.362,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 345.595,\n", + " 388.571,\n", + " 639.748,\n", + " 708.315,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 308.518,\n", + " 188.634,\n", + " 207.793,\n", + " 271.046,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 329.454,\n", + " 249.735,\n", + " 233.447,\n", + " 215.966,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 319.506,\n", + " 267.735,\n", + " 260.868,\n", + " 258.11,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 320.33,\n", + " 274.635,\n", + " 271.74,\n", + " 268.684,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 341.994,\n", + " 279.7,\n", + " 276.276,\n", + " 274.464,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 343.043,\n", + " 282.517,\n", + " 279.985,\n", + " 277.087,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 344.1,\n", + " 283.742,\n", + " 281.84,\n", + " 279.701]},\n", + " 'outputFormat': 'array',\n", + " 'propertyCode': 'pressure',\n", + " 'sensorCategoryCode': 'pressure',\n", + " 'sensorCode': 'Pressure',\n", + " 'sensorName': 'Pressure',\n", + " 'unitOfMeasure': 'decibar'},\n", + " {'actualSamples': 3,\n", + " 'data': {'qaqcFlags': [0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0],\n", + " 'sampleTimes': ['2009-08-27T00:53:29.800Z',\n", + " '2009-08-27T00:53:30.043Z',\n", + " '2009-08-27T00:53:30.289Z',\n", + " '2009-08-27T00:53:30.539Z',\n", + " '2009-08-27T00:53:30.789Z',\n", + " '2009-08-27T00:53:31.039Z',\n", + " '2009-08-27T00:53:31.289Z',\n", + " '2009-08-27T00:53:31.539Z',\n", + " '2009-08-27T00:53:31.789Z',\n", + " '2009-08-27T00:53:32.039Z',\n", + " '2009-08-27T00:54:30.585Z',\n", + " '2009-08-27T00:54:30.834Z',\n", + " '2009-08-27T00:54:31.084Z',\n", + " '2009-08-27T00:54:31.333Z',\n", + " '2009-08-27T00:54:31.583Z',\n", + " '2009-08-27T00:54:31.833Z',\n", + " '2009-08-27T00:54:32.083Z',\n", + " '2009-08-27T00:54:32.333Z',\n", + " '2009-08-27T00:54:32.583Z',\n", + " '2009-08-27T00:54:32.833Z',\n", + " '2009-08-27T00:55:30.578Z',\n", + " '2009-08-27T00:55:30.827Z',\n", + " '2009-08-27T00:55:31.077Z',\n", + " '2009-08-27T00:55:31.325Z',\n", + " '2009-08-27T00:55:31.575Z',\n", + " '2009-08-27T00:55:31.825Z',\n", + " '2009-08-27T00:55:32.075Z',\n", + " '2009-08-27T00:55:32.325Z',\n", + " '2009-08-27T00:55:32.575Z',\n", + " '2009-08-27T00:55:32.825Z',\n", + " '2009-08-27T00:56:30.597Z',\n", + " '2009-08-27T00:56:30.847Z',\n", + " '2009-08-27T00:56:31.033Z',\n", + " '2009-08-27T00:56:31.225Z',\n", + " '2009-08-27T00:56:31.475Z',\n", + " '2009-08-27T00:56:31.725Z',\n", + " '2009-08-27T00:56:31.975Z',\n", + " '2009-08-27T00:56:32.225Z',\n", + " '2009-08-27T00:56:32.475Z',\n", + " '2009-08-27T00:56:32.725Z',\n", + " '2009-08-27T00:57:30.380Z',\n", + " '2009-08-27T00:57:30.631Z',\n", + " '2009-08-27T00:57:30.882Z',\n", + " '2009-08-27T00:57:31.131Z',\n", + " '2009-08-27T00:57:31.381Z',\n", + " '2009-08-27T00:57:31.631Z',\n", + " '2009-08-27T00:57:31.881Z',\n", + " '2009-08-27T00:57:32.131Z',\n", + " '2009-08-27T00:57:32.381Z',\n", + " '2009-08-27T00:57:32.631Z',\n", + " '2009-08-27T00:58:30.387Z',\n", + " '2009-08-27T00:58:30.638Z',\n", + " '2009-08-27T00:58:30.889Z',\n", + " '2009-08-27T00:58:31.141Z',\n", + " '2009-08-27T00:58:31.391Z',\n", + " '2009-08-27T00:58:31.641Z',\n", + " '2009-08-27T00:58:31.891Z',\n", + " '2009-08-27T00:58:32.141Z',\n", + " '2009-08-27T00:58:32.391Z',\n", + " '2009-08-27T00:58:32.641Z',\n", + " '2009-08-27T00:59:30.389Z',\n", + " '2009-08-27T00:59:30.639Z',\n", + " '2009-08-27T00:59:30.888Z',\n", + " '2009-08-27T00:59:31.136Z',\n", + " '2009-08-27T00:59:31.386Z',\n", + " '2009-08-27T00:59:31.636Z',\n", + " '2009-08-27T00:59:31.886Z',\n", + " '2009-08-27T00:59:32.136Z',\n", + " '2009-08-27T00:59:32.386Z',\n", + " '2009-08-27T00:59:32.636Z',\n", + " '2009-08-27T01:00:30.407Z',\n", + " '2009-08-27T01:00:30.657Z',\n", + " '2009-08-27T01:00:30.907Z',\n", + " '2009-08-27T01:00:31.160Z',\n", + " '2009-08-27T01:00:31.410Z',\n", + " '2009-08-27T01:00:31.660Z',\n", + " '2009-08-27T01:00:31.910Z',\n", + " '2009-08-27T01:00:32.160Z',\n", + " '2009-08-27T01:00:32.410Z',\n", + " '2009-08-27T01:00:32.660Z',\n", + " '2009-08-27T01:01:30.408Z',\n", + " '2009-08-27T01:01:30.657Z',\n", + " '2009-08-27T01:01:30.909Z',\n", + " '2009-08-27T01:01:31.159Z',\n", + " '2009-08-27T01:01:31.409Z',\n", + " '2009-08-27T01:01:31.659Z',\n", + " '2009-08-27T01:01:31.909Z',\n", + " '2009-08-27T01:01:32.159Z',\n", + " '2009-08-27T01:01:32.409Z',\n", + " '2009-08-27T01:01:32.659Z',\n", + " '2009-08-27T01:02:30.424Z',\n", + " '2009-08-27T01:02:30.674Z',\n", + " '2009-08-27T01:02:30.923Z',\n", + " '2009-08-27T01:02:31.173Z'],\n", + " 'values': [34.0584,\n", + " 34.0562,\n", + " 34.0601,\n", + " 34.06,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 34.0262,\n", + " 34.0414,\n", + " 34.0432,\n", + " 34.0523,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 34.0837,\n", + " 34.0627,\n", + " 33.9396,\n", + " 33.9069,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 34.1021,\n", + " 34.1618,\n", + " 34.152,\n", + " 34.1208,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 34.0914,\n", + " 34.1315,\n", + " 34.1392,\n", + " 34.1472,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 34.0973,\n", + " 34.1225,\n", + " 34.1254,\n", + " 34.1267,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 34.0975,\n", + " 34.12,\n", + " 34.1205,\n", + " 34.1217,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 34.0859,\n", + " 34.1166,\n", + " 34.1181,\n", + " 34.1183,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 34.085,\n", + " 34.1149,\n", + " 34.1161,\n", + " 34.1175,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 34.0847,\n", + " 34.1148,\n", + " 34.1156,\n", + " 34.1165]},\n", + " 'outputFormat': 'array',\n", + " 'propertyCode': 'salinity',\n", + " 'sensorCategoryCode': 'salinity',\n", + " 'sensorCode': 'salinity',\n", + " 'sensorName': 'Practical Salinity',\n", + " 'unitOfMeasure': 'psu'},\n", + " {'actualSamples': 3,\n", + " 'data': {'qaqcFlags': [0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0],\n", + " 'sampleTimes': ['2009-08-27T00:53:29.800Z',\n", + " '2009-08-27T00:53:30.043Z',\n", + " '2009-08-27T00:53:30.289Z',\n", + " '2009-08-27T00:53:30.539Z',\n", + " '2009-08-27T00:53:30.789Z',\n", + " '2009-08-27T00:53:31.039Z',\n", + " '2009-08-27T00:53:31.289Z',\n", + " '2009-08-27T00:53:31.539Z',\n", + " '2009-08-27T00:53:31.789Z',\n", + " '2009-08-27T00:53:32.039Z',\n", + " '2009-08-27T00:54:30.585Z',\n", + " '2009-08-27T00:54:30.834Z',\n", + " '2009-08-27T00:54:31.084Z',\n", + " '2009-08-27T00:54:31.333Z',\n", + " '2009-08-27T00:54:31.583Z',\n", + " '2009-08-27T00:54:31.833Z',\n", + " '2009-08-27T00:54:32.083Z',\n", + " '2009-08-27T00:54:32.333Z',\n", + " '2009-08-27T00:54:32.583Z',\n", + " '2009-08-27T00:54:32.833Z',\n", + " '2009-08-27T00:55:30.578Z',\n", + " '2009-08-27T00:55:30.827Z',\n", + " '2009-08-27T00:55:31.077Z',\n", + " '2009-08-27T00:55:31.325Z',\n", + " '2009-08-27T00:55:31.575Z',\n", + " '2009-08-27T00:55:31.825Z',\n", + " '2009-08-27T00:55:32.075Z',\n", + " '2009-08-27T00:55:32.325Z',\n", + " '2009-08-27T00:55:32.575Z',\n", + " '2009-08-27T00:55:32.825Z',\n", + " '2009-08-27T00:56:30.597Z',\n", + " '2009-08-27T00:56:30.847Z',\n", + " '2009-08-27T00:56:31.033Z',\n", + " '2009-08-27T00:56:31.225Z',\n", + " '2009-08-27T00:56:31.475Z',\n", + " '2009-08-27T00:56:31.725Z',\n", + " '2009-08-27T00:56:31.975Z',\n", + " '2009-08-27T00:56:32.225Z',\n", + " '2009-08-27T00:56:32.475Z',\n", + " '2009-08-27T00:56:32.725Z',\n", + " '2009-08-27T00:57:30.380Z',\n", + " '2009-08-27T00:57:30.631Z',\n", + " '2009-08-27T00:57:30.882Z',\n", + " '2009-08-27T00:57:31.131Z',\n", + " '2009-08-27T00:57:31.381Z',\n", + " '2009-08-27T00:57:31.631Z',\n", + " '2009-08-27T00:57:31.881Z',\n", + " '2009-08-27T00:57:32.131Z',\n", + " '2009-08-27T00:57:32.381Z',\n", + " '2009-08-27T00:57:32.631Z',\n", + " '2009-08-27T00:58:30.387Z',\n", + " '2009-08-27T00:58:30.638Z',\n", + " '2009-08-27T00:58:30.889Z',\n", + " '2009-08-27T00:58:31.141Z',\n", + " '2009-08-27T00:58:31.391Z',\n", + " '2009-08-27T00:58:31.641Z',\n", + " '2009-08-27T00:58:31.891Z',\n", + " '2009-08-27T00:58:32.141Z',\n", + " '2009-08-27T00:58:32.391Z',\n", + " '2009-08-27T00:58:32.641Z',\n", + " '2009-08-27T00:59:30.389Z',\n", + " '2009-08-27T00:59:30.639Z',\n", + " '2009-08-27T00:59:30.888Z',\n", + " '2009-08-27T00:59:31.136Z',\n", + " '2009-08-27T00:59:31.386Z',\n", + " '2009-08-27T00:59:31.636Z',\n", + " '2009-08-27T00:59:31.886Z',\n", + " '2009-08-27T00:59:32.136Z',\n", + " '2009-08-27T00:59:32.386Z',\n", + " '2009-08-27T00:59:32.636Z',\n", + " '2009-08-27T01:00:30.407Z',\n", + " '2009-08-27T01:00:30.657Z',\n", + " '2009-08-27T01:00:30.907Z',\n", + " '2009-08-27T01:00:31.160Z',\n", + " '2009-08-27T01:00:31.410Z',\n", + " '2009-08-27T01:00:31.660Z',\n", + " '2009-08-27T01:00:31.910Z',\n", + " '2009-08-27T01:00:32.160Z',\n", + " '2009-08-27T01:00:32.410Z',\n", + " '2009-08-27T01:00:32.660Z',\n", + " '2009-08-27T01:01:30.408Z',\n", + " '2009-08-27T01:01:30.657Z',\n", + " '2009-08-27T01:01:30.909Z',\n", + " '2009-08-27T01:01:31.159Z',\n", + " '2009-08-27T01:01:31.409Z',\n", + " '2009-08-27T01:01:31.659Z',\n", + " '2009-08-27T01:01:31.909Z',\n", + " '2009-08-27T01:01:32.159Z',\n", + " '2009-08-27T01:01:32.409Z',\n", + " '2009-08-27T01:01:32.659Z',\n", + " '2009-08-27T01:02:30.424Z',\n", + " '2009-08-27T01:02:30.674Z',\n", + " '2009-08-27T01:02:30.923Z',\n", + " '2009-08-27T01:02:31.173Z'],\n", + " 'values': [26.83482199383866,\n", + " 26.833135110028707,\n", + " 26.836124468444496,\n", + " 26.83604313520459,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 26.810075942027424,\n", + " 26.821797794790882,\n", + " 26.82314384078154,\n", + " 26.830136239442254,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 26.854376133816686,\n", + " 26.838231537027696,\n", + " 26.743609425131126,\n", + " 26.7185395999486,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 26.868567078853857,\n", + " 26.914495190317666,\n", + " 26.906947076811548,\n", + " 26.88296826678652,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 26.860317002403917,\n", + " 26.89120368114618,\n", + " 26.897095706605114,\n", + " 26.90329931409292,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 26.865147630873935,\n", + " 26.884512111930007,\n", + " 26.886732587094002,\n", + " 26.887743736117272,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 26.86554888761725,\n", + " 26.88285528766187,\n", + " 26.88323232524158,\n", + " 26.884161211195078,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 26.856607287764973,\n", + " 26.88023273309932,\n", + " 26.881382567675928,\n", + " 26.88153385031319,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 26.855906804638153,\n", + " 26.878868945131217,\n", + " 26.879828027172834,\n", + " 26.8808920177878,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 26.85576714051558,\n", + " 26.878938635334407,\n", + " 26.879538725086377,\n", + " 26.880227723996086]},\n", + " 'outputFormat': 'array',\n", + " 'propertyCode': 'sigmatheta',\n", + " 'sensorCategoryCode': 'sigma_theta',\n", + " 'sensorCode': 'SIGMA_THETA',\n", + " 'sensorName': 'Sigma-theta (0 dbar)',\n", + " 'unitOfMeasure': 'kg/m3'},\n", + " {'actualSamples': 3,\n", + " 'data': {'qaqcFlags': [0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 9,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0],\n", + " 'sampleTimes': ['2009-08-27T00:53:29.800Z',\n", + " '2009-08-27T00:53:30.043Z',\n", + " '2009-08-27T00:53:30.289Z',\n", + " '2009-08-27T00:53:30.539Z',\n", + " '2009-08-27T00:53:30.789Z',\n", + " '2009-08-27T00:53:31.039Z',\n", + " '2009-08-27T00:53:31.289Z',\n", + " '2009-08-27T00:53:31.539Z',\n", + " '2009-08-27T00:53:31.789Z',\n", + " '2009-08-27T00:53:32.039Z',\n", + " '2009-08-27T00:54:30.585Z',\n", + " '2009-08-27T00:54:30.834Z',\n", + " '2009-08-27T00:54:31.084Z',\n", + " '2009-08-27T00:54:31.333Z',\n", + " '2009-08-27T00:54:31.583Z',\n", + " '2009-08-27T00:54:31.833Z',\n", + " '2009-08-27T00:54:32.083Z',\n", + " '2009-08-27T00:54:32.333Z',\n", + " '2009-08-27T00:54:32.583Z',\n", + " '2009-08-27T00:54:32.833Z',\n", + " '2009-08-27T00:55:30.578Z',\n", + " '2009-08-27T00:55:30.827Z',\n", + " '2009-08-27T00:55:31.077Z',\n", + " '2009-08-27T00:55:31.325Z',\n", + " '2009-08-27T00:55:31.575Z',\n", + " '2009-08-27T00:55:31.825Z',\n", + " '2009-08-27T00:55:32.075Z',\n", + " '2009-08-27T00:55:32.325Z',\n", + " '2009-08-27T00:55:32.575Z',\n", + " '2009-08-27T00:55:32.825Z',\n", + " '2009-08-27T00:56:30.597Z',\n", + " '2009-08-27T00:56:30.847Z',\n", + " '2009-08-27T00:56:31.033Z',\n", + " '2009-08-27T00:56:31.225Z',\n", + " '2009-08-27T00:56:31.475Z',\n", + " '2009-08-27T00:56:31.725Z',\n", + " '2009-08-27T00:56:31.975Z',\n", + " '2009-08-27T00:56:32.225Z',\n", + " '2009-08-27T00:56:32.475Z',\n", + " '2009-08-27T00:56:32.725Z',\n", + " '2009-08-27T00:57:30.380Z',\n", + " '2009-08-27T00:57:30.631Z',\n", + " '2009-08-27T00:57:30.882Z',\n", + " '2009-08-27T00:57:31.131Z',\n", + " '2009-08-27T00:57:31.381Z',\n", + " '2009-08-27T00:57:31.631Z',\n", + " '2009-08-27T00:57:31.881Z',\n", + " '2009-08-27T00:57:32.131Z',\n", + " '2009-08-27T00:57:32.381Z',\n", + " '2009-08-27T00:57:32.631Z',\n", + " '2009-08-27T00:58:30.387Z',\n", + " '2009-08-27T00:58:30.638Z',\n", + " '2009-08-27T00:58:30.889Z',\n", + " '2009-08-27T00:58:31.141Z',\n", + " '2009-08-27T00:58:31.391Z',\n", + " '2009-08-27T00:58:31.641Z',\n", + " '2009-08-27T00:58:31.891Z',\n", + " '2009-08-27T00:58:32.141Z',\n", + " '2009-08-27T00:58:32.391Z',\n", + " '2009-08-27T00:58:32.641Z',\n", + " '2009-08-27T00:59:30.389Z',\n", + " '2009-08-27T00:59:30.639Z',\n", + " '2009-08-27T00:59:30.888Z',\n", + " '2009-08-27T00:59:31.136Z',\n", + " '2009-08-27T00:59:31.386Z',\n", + " '2009-08-27T00:59:31.636Z',\n", + " '2009-08-27T00:59:31.886Z',\n", + " '2009-08-27T00:59:32.136Z',\n", + " '2009-08-27T00:59:32.386Z',\n", + " '2009-08-27T00:59:32.636Z',\n", + " '2009-08-27T01:00:30.407Z',\n", + " '2009-08-27T01:00:30.657Z',\n", + " '2009-08-27T01:00:30.907Z',\n", + " '2009-08-27T01:00:31.160Z',\n", + " '2009-08-27T01:00:31.410Z',\n", + " '2009-08-27T01:00:31.660Z',\n", + " '2009-08-27T01:00:31.910Z',\n", + " '2009-08-27T01:00:32.160Z',\n", + " '2009-08-27T01:00:32.410Z',\n", + " '2009-08-27T01:00:32.660Z',\n", + " '2009-08-27T01:01:30.408Z',\n", + " '2009-08-27T01:01:30.657Z',\n", + " '2009-08-27T01:01:30.909Z',\n", + " '2009-08-27T01:01:31.159Z',\n", + " '2009-08-27T01:01:31.409Z',\n", + " '2009-08-27T01:01:31.659Z',\n", + " '2009-08-27T01:01:31.909Z',\n", + " '2009-08-27T01:01:32.159Z',\n", + " '2009-08-27T01:01:32.409Z',\n", + " '2009-08-27T01:01:32.659Z',\n", + " '2009-08-27T01:02:30.424Z',\n", + " '2009-08-27T01:02:30.674Z',\n", + " '2009-08-27T01:02:30.923Z',\n", + " '2009-08-27T01:02:31.173Z'],\n", + " 'values': [5.8232,\n", + " 5.8232,\n", + " 5.8231,\n", + " 5.8231,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 5.8231,\n", + " 5.8228,\n", + " 5.823,\n", + " 5.823,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 5.8224,\n", + " 5.8224,\n", + " 5.8226,\n", + " 5.8224,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 5.8221,\n", + " 5.8222,\n", + " 5.8222,\n", + " 5.822,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 5.8222,\n", + " 5.822,\n", + " 5.8222,\n", + " 5.8217,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 5.82,\n", + " 5.8201,\n", + " 5.8201,\n", + " 5.82,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 5.8181,\n", + " 5.8181,\n", + " 5.818,\n", + " 5.8179,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 5.8181,\n", + " 5.818,\n", + " 5.818,\n", + " 5.8179,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 5.8181,\n", + " 5.8184,\n", + " 5.8181,\n", + " 5.8182,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " nan,\n", + " 5.8174,\n", + " 5.8173,\n", + " 5.8174,\n", + " 5.8174]},\n", + " 'outputFormat': 'array',\n", + " 'propertyCode': 'seawatertemperature',\n", + " 'sensorCategoryCode': 'temperature',\n", + " 'sensorCode': 'Temperature',\n", + " 'sensorName': 'Temperature',\n", + " 'unitOfMeasure': 'C'}]" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 7 + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-11-10T21:10:51.298653Z", + "start_time": "2025-11-10T21:10:51.282654Z" + } + }, + "cell_type": "markdown", + "source": "## Bad Data Request Example", + "id": "5c3b637cf29f849f" + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-11-10T21:17:12.631455Z", + "start_time": "2025-11-10T21:17:12.280322Z" + } + }, + "cell_type": "code", + "source": [ + "data = onc.getScalardata({'locationCode':'BACVP',\n", + " 'deviceCategoryCode':'CTD',\n", + " 'dateFrom': '2009-08-15T00:00:00.000Z',\n", + " 'dateTo': '2009-08-17T23:59:59.999Z',\n", + " 'rowLimit': 10}, allPages = True)\n", + "\n", + "if isinstance(data, dict):\n", + " print(data)\n", + "else: # Assume it is a requests.Response object.\n", + " print(data.status_code)" + ], + "id": "d27aaa83f916109e", + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2025-11-10T21:17:12.620Z | onc-service | INFO | Requested: https://data.oceannetworks.ca/api/scalardata?locationCode=BACVP&deviceCategoryCode=CTD&dateFrom=2009-08-15T00%3A00%3A00.000Z&dateTo=2009-08-17T23%3A59%3A59.999Z&rowLimit=10&method=getByLocation&token=REDACTED\n", + "2025-11-10T21:17:12.620Z | onc-service | DEBUG | Response received in 0.341 seconds.\n", + "2025-11-10T21:17:12.620Z | onc-service | ERROR | HTTP Response: Bad Request (400)\n", + "2025-11-10T21:17:12.620Z | onc-service | ERROR | (API Error Code 127) A device with category CTD was deployed at location BACVP but not during the provided time range (20090815T000000.000Z to 20090817T235959.999Z). The deployment service can be used to determine a valid time range: https://data.oceannetworks.ca/api/deployments?locationCode=BACVP&deviceCategoryCode=CTD&token=REDACTED for query parameter(s) 'locationCode, deviceCategoryCode, dateFrom, dateTo'.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "400\n" + ] + } + ], + "execution_count": 6 + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 2 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython2", + "version": "2.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/src/onc/modules/_Messages.py b/src/onc/modules/_Messages.py new file mode 100644 index 0000000..19b1ada --- /dev/null +++ b/src/onc/modules/_Messages.py @@ -0,0 +1,86 @@ +import logging +import re +import requests +import time + +REQ_MSG = "Requested: {}" # get request url +RESPONSE_TIME_MSG = "Response received in {} seconds." # requests.elapsed value. +RESPONSE_MSG = "HTTP Response: {} ({})" # Brief description, status code +MULTIPAGE_MSG = ("The requested data quantity is greater than the " + "supplied row limit and will be downloaded over multiple requests.") + + +def setup_logger(logger_name: str = 'onc-client', + level: int | str = 'DEBUG') -> logging.Logger: + """ + Set up a logger object for displaying verbose messages to console. + + :param logger_name: The unique logger name to use. Can be shared between modules + :param level: The logging level to use. Default is 2, which corresponds to DEBUG. + :return: The configured logging.Logger object. + """ + + logger = logging.getLogger(logger_name) + logger.propagate = False + if not logger.handlers: + logger.setLevel(logging.DEBUG) + console = logging.StreamHandler() + console.setLevel(level) + + # Set the logging format. + dtfmt = '%Y-%m-%dT%H:%M:%S' + strfmt = f'%(asctime)s.%(msecs)03dZ | %(name)-12s | %(levelname)-8s | %(message)s' + #strfmt = f'%(asctime)s.%(msecs)03dZ | %(levelname)-8s | %(message)s' # Use this if you don't want to include logger name. + fmt = logging.Formatter(strfmt, datefmt=dtfmt) + fmt.converter = time.gmtime + + console.setFormatter(fmt) + logger.addHandler(console) + return logger + + +def scrub_token(input: str) -> str: + """ + Replace a token in a query URL or other string with the string 'REDACTED' + so that users don't accidentally commit their tokens to public repositories + if ONC Info/Warnings are too verbose. + + :param query_url: An Oceans 3.0 API URL or string with a token query parameter. + :return: A scrubbed url. + """ + token_regex = r'(&token=[a-f0-9-]{36})' + token_qp = re.findall(token_regex, input)[0] + redacted_url = input.replace(token_qp, '&token=REDACTED') + return redacted_url + + +def build_error_message(response: requests.Response, redact_token: bool) -> str: + """ + Build an error message from a requests.Response object. + + :param response: A requests.Response object. + :param redact_token: If true, redact tokens before returning an error message. + :return: An error message. + """ + payload = response.json() + if 'message' in payload.keys(): + message = payload['message'] + else: + message = None + + if 'errors' in payload.keys(): + errors = payload['errors'] + error_messages = [] + for error in errors: + emsg = (f"(API Error Code {error['errorCode']}) " + f"{error['errorMessage']} for query parameter(s) " + f"'{error['parameter']}'.") + error_messages.append(emsg) + error_message = '\n'.join(error_messages) + else: + error_message = None + msg = '\n'.join([m for m in (message, error_message) if m is not None]) + if redact_token is True and 'token=' in msg: + msg = scrub_token(msg) + return msg + diff --git a/src/onc/modules/_MultiPage.py b/src/onc/modules/_MultiPage.py index 6369773..9a3318d 100644 --- a/src/onc/modules/_MultiPage.py +++ b/src/onc/modules/_MultiPage.py @@ -8,12 +8,23 @@ from ._util import _formatDuration +from onc.modules._Messages import (setup_logger, MULTIPAGE_MSG, + build_error_message, + scrub_token, + REQ_MSG, + RESPONSE_TIME_MSG, + RESPONSE_MSG) + + # Handles data multi-page downloads (scalardata, rawdata, archivefiles) class _MultiPage: - def __init__(self, parent: object): + def __init__(self, parent: object, verbosity: bool, raise_http_errors: bool): self.parent = weakref.ref(parent) self.result = None + self.raise_http_errors = raise_http_errors + self.__log = setup_logger('onc-multi', verbosity) + def getAllPages(self, service: str, url: str, filters: dict): """ @@ -30,48 +41,42 @@ def getAllPages(self, service: str, url: str, filters: dict): # download first page start = time() response, responseTime = self._doPageRequest(url, filters, service, extension) - rNext = response["next"] - - if rNext is not None: - print( - "Data quantity is greater than the row limit and", - "will be downloaded in multiple pages.", - ) - - pageCount = 1 - pageEstimate = self._estimatePages(response, service) - if pageEstimate > 0: - # Exclude the first page when calculating the time estimation - timeEstimate = _formatDuration((pageEstimate - 1) * responseTime) - print( - f"Downloading time for the first page: {humanize.naturaldelta(responseTime)}" # noqa: E501 - ) - print(f"Estimated approx. {pageEstimate} pages in total.") - print( - f"Estimated approx. {timeEstimate} to complete for the rest of the pages." # noqa: E501 - ) - # keep downloading pages until next is None - print("") - while rNext is not None: - pageCount += 1 - rowCount = self._rowCount(response, service) + if isinstance(response,dict): + rNext = response["next"] - print(f" ({rowCount} samples) Downloading page {pageCount}...") - nextResponse, nextTime = self._doPageRequest( - url, rNext["parameters"], service, extension - ) - rNext = nextResponse["next"] + if rNext is not None: + self.__log.info("The requested data quantity is greater than the supplied " + "row limit and will be downloaded over multiple requests.") + + pageCount = 1 + pageEstimate = self._estimatePages(response, service) + if pageEstimate > 0: + # Exclude the first page when calculating the time estimation + timeEstimate = _formatDuration((pageEstimate - 1) * responseTime) + self.__log.debug(f'Download time for page {pageCount}: {round(responseTime,2)} seconds') + self.__log.info(f'Est. number of pages remaining for download: {pageEstimate-1}') + self.__log.info(f'Est. number of seconds to download remaining data: {timeEstimate}') + + # keep downloading pages until next is None + while rNext is not None: + pageCount += 1 + rowCount = self._rowCount(response, service) + + self.__log.debug(f"Submitting request for page {pageCount} ({rowCount} samples)...") + + nextResponse, nextTime = self._doPageRequest( + url, rNext["parameters"], service, extension + ) + rNext = nextResponse["next"] + + # concatenate new data obtained + self._catenateData(response, nextResponse, service) - # concatenate new data obtained - self._catenateData(response, nextResponse, service) + totalTime = _formatDuration(time() - start) - totalTime = _formatDuration(time() - start) - print( - f" ({self._rowCount(response, service):d} samples)" - f" Completed in {totalTime}." - ) - response["next"] = None + self.__log.info(f"Downloaded {self._rowCount(response, service):d} total samples in {totalTime}.") + response["next"] = None return response diff --git a/src/onc/modules/_OncArchive.py b/src/onc/modules/_OncArchive.py index 9129a99..dbfbd85 100644 --- a/src/onc/modules/_OncArchive.py +++ b/src/onc/modules/_OncArchive.py @@ -14,8 +14,9 @@ class _OncArchive(_OncService): Methods that wrap the API archivefiles service """ - def __init__(self, parent: object): - super().__init__(parent) + def __init__(self, parent: object, verbosity: str, redact_token: str, raise_http_errors: bool): + super().__init__(parent, verbosity, redact_token, raise_http_errors) + def getArchivefileByLocation(self, filters: dict, allPages: bool): """ diff --git a/src/onc/modules/_OncDelivery.py b/src/onc/modules/_OncDelivery.py index 616ce24..15e75a7 100644 --- a/src/onc/modules/_OncDelivery.py +++ b/src/onc/modules/_OncDelivery.py @@ -16,8 +16,8 @@ class _OncDelivery(_OncService): Methods that wrap the API data product delivery services """ - def __init__(self, parent: object): - super().__init__(parent) + def __init__(self, parent: object, verbosity: str, redact_token: str, raise_http_errors: bool): + super().__init__(parent, verbosity, redact_token, raise_http_errors) # Default seconds to wait between consecutive download tries of a file # (when no estimate processing time is available) diff --git a/src/onc/modules/_OncDiscovery.py b/src/onc/modules/_OncDiscovery.py index 8ae32f8..c6471c1 100644 --- a/src/onc/modules/_OncDiscovery.py +++ b/src/onc/modules/_OncDiscovery.py @@ -7,8 +7,9 @@ class _OncDiscovery(_OncService): locations, deployments, devices, deviceCategories, properties, dataProducts """ - def __init__(self, parent: object): - super().__init__(parent) + def __init__(self, parent: object, verbosity: str, redact_token: str, raise_http_errors: bool): + super().__init__(parent, verbosity, redact_token, raise_http_errors) + def _discoveryRequest(self, filters: dict, service: str): url = self._serviceUrl(service) diff --git a/src/onc/modules/_OncRealTime.py b/src/onc/modules/_OncRealTime.py index 5ec55cb..f8c78ba 100644 --- a/src/onc/modules/_OncRealTime.py +++ b/src/onc/modules/_OncRealTime.py @@ -9,8 +9,8 @@ class _OncRealTime(_OncService): Near real-time services methods """ - def __init__(self, config: dict): - super().__init__(config) + def __init__(self, config: dict, verbosity: str, redact_token: str, raise_http_errors: bool): + super().__init__(config, verbosity, redact_token, raise_http_errors) def getScalardataByLocation(self, filters: dict, allPages: bool): """ @@ -90,7 +90,7 @@ def _getDirectAllPages(self, filters: dict, service: str, allPages: bool) -> Any filters["sensorCategoryCodes"] = ",".join(filters["sensorCategoryCodes"]) if allPages: - mp = _MultiPage(self) + mp = _MultiPage(self, self.verbosity, self.raise_http_errors) result = mp.getAllPages(service, url, filters) else: result = self._doRequest(url, filters) diff --git a/src/onc/modules/_OncService.py b/src/onc/modules/_OncService.py index 02eb532..c8a251c 100644 --- a/src/onc/modules/_OncService.py +++ b/src/onc/modules/_OncService.py @@ -7,6 +7,12 @@ import requests from ._util import _createErrorMessage, _formatDuration +from onc.modules._Messages import (setup_logger, + build_error_message, + scrub_token, + REQ_MSG, + RESPONSE_TIME_MSG, + RESPONSE_MSG) logging.basicConfig(format="%(levelname)s: %(message)s") @@ -16,8 +22,16 @@ class _OncService: Provides common configuration and functionality to Onc service classes (children) """ - def __init__(self, parent: object): + def __init__(self, parent: object, + verbosity: str, + redact_token: bool, + raise_http_errors: bool): self.parent = weakref.ref(parent) + self.redact_token = redact_token + self.raise_http_errors = raise_http_errors + self.verbosity = verbosity + + self.__log = setup_logger('onc-service', level = verbosity) def _doRequest(self, url: str, filters: dict | None = None, getTime: bool = False): """ @@ -44,48 +58,60 @@ def _doRequest(self, url: str, filters: dict | None = None, getTime: bool = Fals filters["token"] = self._config("token") timeout = self._config("timeout") - txtParams = parse.unquote(parse.urlencode(filters)) - self._log(f"Requesting URL:\n{url}?{txtParams}") - - start = time() response = requests.get(url, filters, timeout=timeout) - responseTime = time() - start - if response.ok: - jsonResult = response.json() + if self.redact_token is True: + try: + response_url = scrub_token(response.url) + except: + response_url = response.url else: - status = response.status_code - if status in [400, 401]: - msg = _createErrorMessage(response) - raise requests.HTTPError(msg) - else: - response.raise_for_status() - self._log(f"Web Service response time: {_formatDuration(responseTime)}") - - # Log warning messages only when showWarning is True - # and jsonResult["messages"] is not an empty list - if ( - self._config("showWarning") - and "messages" in jsonResult - and jsonResult["messages"] - ): - long_message = "\n".join( - [f"* {message}" for message in jsonResult["messages"]] - ) + response_url = response.url - filters_without_token = filters.copy() - del filters_without_token["token"] - filters_str = pprint.pformat(filters_without_token) + # Log the url the user submitted. + self.__log.info(REQ_MSG.format(response_url)) - logging.warning( - f"When calling {url} with filters\n{filters_str},\n" - f"there are several warning messages:\n{long_message}\n" - ) + # Display the time it took for ONC to respond in seconds. + # The requests.Response.elapsed value is a datetime.timedelta object. + responseTime = round(response.elapsed.total_seconds(),3) # To milliseconds. + self.__log.debug(RESPONSE_TIME_MSG.format(responseTime)) + + json_response = response.json() - if getTime: - return jsonResult, responseTime + if response.status_code == requests.codes.ok: + self.__log.info(RESPONSE_MSG.format("OK", response.status_code)) + if getTime is True: + return json_response, responseTime + else: + return json_response else: - return jsonResult + if response.status_code == requests.codes.not_found: + self.__log.error(RESPONSE_MSG.format("Not Found", + response.status_code)) + elif response.status_code == requests.codes.bad: + self.__log.error(RESPONSE_MSG.format("Bad Request", + response.status_code)) + elif response.status_code == requests.codes.unauthorized: + self.__log.error(RESPONSE_MSG.format("Unauthorized Request", + response.status_code)) + elif response.status_code == requests.codes.internal_server_error: + self.__log.error(RESPONSE_MSG.format("Internal Server Error", + response.status_code)) + else: + self.__log.error(RESPONSE_MSG.format('Error',response.status_code)) + + self.__log.error(build_error_message(response, + self.redact_token)) + + if self.raise_http_errors is True: + response.raise_for_status() + + else: + if getTime is True: + return response, responseTime + else: + return response + def _serviceUrl(self, service: str): """ @@ -115,14 +141,6 @@ def _serviceUrl(self, service: str): return "" - def _log(self, message: str): - """ - Prints message to console only when self.showInfo is true - @param message: String - """ - if self._config("showInfo"): - print(message) - def _config(self, key: str): """ Returns a property from the parent (ONC class) @@ -145,3 +163,4 @@ def _delegateByFilters(self, byDevice, byLocation, **kwargs): "'locationCode' and 'deviceCategoryCode', " "or a 'deviceCode' present." ) + diff --git a/src/onc/modules/_util.py b/src/onc/modules/_util.py index 16db3fb..e867205 100644 --- a/src/onc/modules/_util.py +++ b/src/onc/modules/_util.py @@ -76,6 +76,7 @@ def _createErrorMessage(response: requests.Response) -> str: elif status == 401: return ( f"Status 401 - Unauthorized: {response.url}\n" + "Please check that your Web Services API token is valid. " "Find your token in your registered profile at " "https://data.oceannetworks.ca." diff --git a/src/onc/onc.py b/src/onc/onc.py index 42b1e85..328abb2 100644 --- a/src/onc/onc.py +++ b/src/onc/onc.py @@ -10,6 +10,7 @@ from onc.modules._OncDiscovery import _OncDiscovery from onc.modules._OncRealTime import _OncRealTime +from onc.modules._Messages import setup_logger class ONC: """ @@ -52,11 +53,12 @@ class ONC: def __init__( self, token: str | None = None, - production: bool = True, - showInfo: bool = False, - showWarning: bool = True, outPath: str | Path = "output", + verbosity: str = "INFO", + raise_http_errors: bool = True, + redact_token: bool = False, timeout: int = 60, + production: bool = True, ): if token is None or token == "": token = os.environ.get("ONC_TOKEN") @@ -65,18 +67,32 @@ def __init__( "ONC API token is required. Please provide it as the first argument, " "or set it as the environment variable 'ONC_TOKEN'." ) + + self.verbosity = verbosity.upper() + self.redact_token = redact_token + self.raise_http_errors = raise_http_errors + self.__log = setup_logger('onc-client', self.verbosity) + + if self.verbosity in ['INFO', 'DEBUG']: + self.showInfo = True + self.showWarning = True + elif self.verbosity in ['WARNING', 'ERROR']: + self.showInfo = False + self.showWarning = True + self.token = re.sub(r"[^a-zA-Z0-9\-]+", "", token) - self.showInfo = showInfo - self.showWarning = showWarning + self.timeout = timeout self.production = production self.outPath = outPath # Create service objects - self.discovery = _OncDiscovery(self) - self.delivery = _OncDelivery(self) - self.realTime = _OncRealTime(self) - self.archive = _OncArchive(self) + self.discovery = _OncDiscovery(self, verbosity, redact_token,raise_http_errors) + self.delivery = _OncDelivery(self, verbosity, redact_token,raise_http_errors) + self.realTime = _OncRealTime(self, verbosity, redact_token,raise_http_errors) + self.archive = _OncArchive(self, verbosity, redact_token,raise_http_errors) + + self.__log.debug("Initialized ONC module.") @property def outPath(self) -> Path: From 36401d79fbe08ed308d5f0da66e2cc3755a9857d Mon Sep 17 00:00:00 2001 From: Kan Fu Date: Wed, 15 Apr 2026 16:19:46 -0700 Subject: [PATCH 2/4] refactor: replace direct console prints with structured logging across ONC modules and update constructor signatures --- src/onc/modules/_DataProductFile.py | 20 +++--- src/onc/modules/_Messages.py | 60 +++++++++++----- src/onc/modules/_MultiPage.py | 24 ++++--- src/onc/modules/_OncArchive.py | 19 +++--- src/onc/modules/_OncDelivery.py | 52 +++++++------- src/onc/modules/_OncDiscovery.py | 4 +- src/onc/modules/_OncRealTime.py | 6 +- src/onc/modules/_OncService.py | 67 +++++++----------- src/onc/modules/_PollLog.py | 45 +++++------- src/onc/onc.py | 102 +++++++++++++++++++++------- 10 files changed, 228 insertions(+), 171 deletions(-) diff --git a/src/onc/modules/_DataProductFile.py b/src/onc/modules/_DataProductFile.py index b3d1e23..b864080 100644 --- a/src/onc/modules/_DataProductFile.py +++ b/src/onc/modules/_DataProductFile.py @@ -15,11 +15,15 @@ def __init__(self, max_retries): class _DataProductFile: """ - Donwloads a single data product file + Downloads a single data product file Is able to poll and wait if required """ - def __init__(self, dpRunId: int, index: str, baseUrl: str, token: str): + def __init__(self, dpRunId: int, index: str, baseUrl: str, token: str, verbosity: str = "INFO"): + self.verbosity = verbosity + # Use child logger 'onc.poll' consistent with _PollLog + from ._Messages import setup_logger + self._log = setup_logger('onc.poll', verbosity) self._retries = 0 self._status = 202 self._downloaded = False @@ -50,7 +54,7 @@ def download( Can poll, wait and retry if the file is not ready to download Return the file information """ - log = _PollLog(True) + log = _PollLog(self.verbosity) self._status = 202 while self._status == 202: # Run timed request @@ -74,9 +78,7 @@ def download( response, outPath, filename, overwrite ) except FileExistsError: - if self._retries > 1: - print("") - print(f' Skipping "{self._filePath}": File already exists.') + self._log.info(f' Skipping "{self._filePath}": File already exists.') self._status = 777 elif self._status == 202: # Still processing, wait and retry @@ -84,10 +86,9 @@ def download( sleep(pollPeriod) elif self._status == 204: # No data found - print(" No data found.") + self._log.info(" No data found.") elif self._status == 404: # Index too high, no more files to download - log.printNewLine() pass elif self._status == 410: # Status 410: gone (file deleted from FTP) @@ -98,7 +99,8 @@ def download( stacklevel=2, ) else: - raise requests.HTTPError(_createErrorMessage(response)) + self._log.error(_createErrorMessage(response)) + response.raise_for_status() return self._status diff --git a/src/onc/modules/_Messages.py b/src/onc/modules/_Messages.py index 19b1ada..9957be4 100644 --- a/src/onc/modules/_Messages.py +++ b/src/onc/modules/_Messages.py @@ -10,48 +10,77 @@ "supplied row limit and will be downloaded over multiple requests.") -def setup_logger(logger_name: str = 'onc-client', - level: int | str = 'DEBUG') -> logging.Logger: + +LEVEL_MAP = { + 'CRITICAL': logging.CRITICAL, + 'ERROR': logging.ERROR, + 'WARNING': logging.WARNING, + 'INFO': logging.INFO, + 'DEBUG': logging.DEBUG, +} + + +class OnclibFormatter(logging.Formatter): + """ + Custom formatter that removes prefix for INFO level logs. + """ + def format(self, record): + if record.levelno == logging.INFO: + return record.getMessage() + return super().format(record) + + +def setup_logger(logger_name: str = 'onc', + level: int | str = 'INFO') -> logging.Logger: """ Set up a logger object for displaying verbose messages to console. :param logger_name: The unique logger name to use. Can be shared between modules - :param level: The logging level to use. Default is 2, which corresponds to DEBUG. + :param level: The logging level to use. Default is 'INFO'. :return: The configured logging.Logger object. """ + # Ensure level is a valid logging level integer + if isinstance(level, str): + level = LEVEL_MAP.get(level.upper(), logging.INFO) + logger = logging.getLogger(logger_name) logger.propagate = False + + # Apply level to the logger itself + logger.setLevel(level) + if not logger.handlers: - logger.setLevel(logging.DEBUG) console = logging.StreamHandler() console.setLevel(level) # Set the logging format. dtfmt = '%Y-%m-%dT%H:%M:%S' strfmt = f'%(asctime)s.%(msecs)03dZ | %(name)-12s | %(levelname)-8s | %(message)s' - #strfmt = f'%(asctime)s.%(msecs)03dZ | %(levelname)-8s | %(message)s' # Use this if you don't want to include logger name. - fmt = logging.Formatter(strfmt, datefmt=dtfmt) + fmt = OnclibFormatter(strfmt, datefmt=dtfmt) fmt.converter = time.gmtime console.setFormatter(fmt) logger.addHandler(console) + else: + # If handlers exist, ensure they have the correct level + for handler in logger.handlers: + handler.setLevel(level) + return logger + def scrub_token(input: str) -> str: """ Replace a token in a query URL or other string with the string 'REDACTED' so that users don't accidentally commit their tokens to public repositories if ONC Info/Warnings are too verbose. - :param query_url: An Oceans 3.0 API URL or string with a token query parameter. + :param input: An Oceans 3.0 API URL or string with a token query parameter. :return: A scrubbed url. """ - token_regex = r'(&token=[a-f0-9-]{36})' - token_qp = re.findall(token_regex, input)[0] - redacted_url = input.replace(token_qp, '&token=REDACTED') - return redacted_url + return re.sub(r"([?&]token=)[a-f0-9-]{36}", r"\1REDACTED", input) def build_error_message(response: requests.Response, redact_token: bool) -> str: @@ -63,13 +92,10 @@ def build_error_message(response: requests.Response, redact_token: bool) -> str: :return: An error message. """ payload = response.json() - if 'message' in payload.keys(): - message = payload['message'] - else: - message = None + message = payload.get("message") - if 'errors' in payload.keys(): - errors = payload['errors'] + if "errors" in payload: + errors = payload["errors"] error_messages = [] for error in errors: emsg = (f"(API Error Code {error['errorCode']}) " diff --git a/src/onc/modules/_MultiPage.py b/src/onc/modules/_MultiPage.py index 9a3318d..61a7ec6 100644 --- a/src/onc/modules/_MultiPage.py +++ b/src/onc/modules/_MultiPage.py @@ -19,12 +19,18 @@ # Handles data multi-page downloads (scalardata, rawdata, archivefiles) class _MultiPage: - def __init__(self, parent: object, verbosity: bool, raise_http_errors: bool): + def __init__(self, parent: object): self.parent = weakref.ref(parent) self.result = None - self.raise_http_errors = raise_http_errors - self.__log = setup_logger('onc-multi', verbosity) + self._log = setup_logger('onc.multi', self._config('verbosity') or 'INFO') + def _config(self, key): + p = self.parent() + if p is None: + return None + if hasattr(p, '_config'): + return p._config(key) + return getattr(p, key, None) def getAllPages(self, service: str, url: str, filters: dict): """ @@ -46,7 +52,7 @@ def getAllPages(self, service: str, url: str, filters: dict): rNext = response["next"] if rNext is not None: - self.__log.info("The requested data quantity is greater than the supplied " + self._log.info("The requested data quantity is greater than the supplied " "row limit and will be downloaded over multiple requests.") pageCount = 1 @@ -54,16 +60,16 @@ def getAllPages(self, service: str, url: str, filters: dict): if pageEstimate > 0: # Exclude the first page when calculating the time estimation timeEstimate = _formatDuration((pageEstimate - 1) * responseTime) - self.__log.debug(f'Download time for page {pageCount}: {round(responseTime,2)} seconds') - self.__log.info(f'Est. number of pages remaining for download: {pageEstimate-1}') - self.__log.info(f'Est. number of seconds to download remaining data: {timeEstimate}') + self._log.info(f'Download time for page {pageCount}: {round(responseTime,2)} seconds') + self._log.info(f'Est. number of pages remaining for download: {pageEstimate-1}') + self._log.info(f'Est. number of seconds to download remaining data: {timeEstimate}') # keep downloading pages until next is None while rNext is not None: pageCount += 1 rowCount = self._rowCount(response, service) - self.__log.debug(f"Submitting request for page {pageCount} ({rowCount} samples)...") + self._log.info(f" Submitting request for page {pageCount} ({rowCount} samples)...") nextResponse, nextTime = self._doPageRequest( url, rNext["parameters"], service, extension @@ -75,7 +81,7 @@ def getAllPages(self, service: str, url: str, filters: dict): totalTime = _formatDuration(time() - start) - self.__log.info(f"Downloaded {self._rowCount(response, service):d} total samples in {totalTime}.") + self._log.info(f" Downloaded {self._rowCount(response, service):d} total samples in {totalTime}.") response["next"] = None return response diff --git a/src/onc/modules/_OncArchive.py b/src/onc/modules/_OncArchive.py index dbfbd85..342a2c6 100644 --- a/src/onc/modules/_OncArchive.py +++ b/src/onc/modules/_OncArchive.py @@ -14,8 +14,8 @@ class _OncArchive(_OncService): Methods that wrap the API archivefiles service """ - def __init__(self, parent: object, verbosity: str, redact_token: str, raise_http_errors: bool): - super().__init__(parent, verbosity, redact_token, raise_http_errors) + def __init__(self, parent: object): + super().__init__(parent) def getArchivefileByLocation(self, filters: dict, allPages: bool): @@ -74,8 +74,9 @@ def downloadArchivefile(self, filename: str = "", overwrite: bool = False): size, downloadTime = saveAsFile(response, outPath, filename, overwrite) else: - msg = _createErrorMessage(response) - raise requests.HTTPError(msg) + from ._Messages import build_error_message + self._log.error(build_error_message(response, self._config("redact_token"))) + response.raise_for_status() # Prepare a readable status txtStatus = "error" @@ -110,7 +111,7 @@ def downloadDirectArchivefile( dataRows = self.getArchivefile(filters, allPages) n = len(dataRows["files"]) - print(f"Obtained a list of {n} files to download.") + self._log.info(f"Obtained a list of {n} files to download.") # Download the files obtained tries = 1 @@ -125,7 +126,7 @@ def downloadDirectArchivefile( fileExists = os.path.exists(filePath) if not fileExists or os.path.getsize(filePath) == 0 or overwrite: - print(f' ({tries} of {n}) Downloading file: "{filename}"') + self._log.info(f' ({tries} of {n}) Downloading file: "{filename}"') downInfo = self.downloadArchivefile(filename, overwrite) size += downInfo["size"] time += downInfo["downloadTime"] @@ -133,7 +134,7 @@ def downloadDirectArchivefile( successes += 1 tries += 1 else: - print(f' Skipping "{filename}": File already exists.') + self._log.info(f' Skipping "{filename}": File already exists.') downInfo = { "url": self.getArchivefileUrl(filename), "status": "skipped", @@ -143,8 +144,8 @@ def downloadDirectArchivefile( } downInfos.append(downInfo) - print(f"{successes} files ({humanize.naturalsize(size)}) downloaded") - print(f"Total Download Time: {_formatDuration(time)}") + self._log.info(f"{successes} files ({humanize.naturalsize(size)}) downloaded") + self._log.info(f"Total Download Time: {_formatDuration(time)}") return { "downloadResults": downInfos, diff --git a/src/onc/modules/_OncDelivery.py b/src/onc/modules/_OncDelivery.py index 15e75a7..b242b2b 100644 --- a/src/onc/modules/_OncDelivery.py +++ b/src/onc/modules/_OncDelivery.py @@ -16,8 +16,8 @@ class _OncDelivery(_OncService): Methods that wrap the API data product delivery services """ - def __init__(self, parent: object, verbosity: str, redact_token: str, raise_http_errors: bool): - super().__init__(parent, verbosity, redact_token, raise_http_errors) + def __init__(self, parent: object): + super().__init__(parent) # Default seconds to wait between consecutive download tries of a file # (when no estimate processing time is available) @@ -56,7 +56,6 @@ def orderDataProduct( ) ) - print("") self._printProductOrderStats(fileList, runData) return self._formatResult(fileList, runData) @@ -88,9 +87,9 @@ def runDataProduct(self, dpRequestId: int, waitComplete: bool): Return a dictionary with information of the run process. """ status = "" - log = _PollLog(True) - print( - f"To cancel the running data product, run 'onc.cancelDataProduct({dpRequestId})'" # noqa: E501 + log = _PollLog(self._config("verbosity")) + self._log.info( + f"To cancel the running data product, run 'onc.cancelDataProduct({dpRequestId})'" ) url = f"{self._config('baseUrl')}api/dataProductDelivery/run" runResult = {"runIds": [], "fileCount": 0, "runTime": 0, "requestCount": 0} @@ -111,7 +110,9 @@ def runDataProduct(self, dpRequestId: int, waitComplete: bool): if response.ok: data = response.json() else: - raise requests.HTTPError(_createErrorMessage(response)) + from ._Messages import build_error_message + self._log.error(build_error_message(response, self._config("redact_token"))) + response.raise_for_status() if waitComplete: status = data[0]["status"] @@ -128,10 +129,6 @@ def runDataProduct(self, dpRequestId: int, waitComplete: bool): runResult["fileCount"] = data[0]["fileCount"] runResult["runTime"] = time() - start - # print a new line after the process finishes - if waitComplete: - print("") - # gather a list of runIds for run in data: runResult["runIds"].append(run["dpRunId"]) @@ -192,9 +189,9 @@ def _downloadProductFiles( # keep increasing index until fileCount or until we get 404 doLoop = True timeout = self._config("timeout") - print(f"\nDownloading data product files with runId {runId}...") + self._log.info(f"Downloading data product files with runId {runId}...") - dpf = _DataProductFile(runId, str(index), baseUrl, token) + dpf = _DataProductFile(runId, str(index), baseUrl, token, self._config("verbosity")) # loop thorugh file indexes while doLoop: @@ -211,7 +208,7 @@ def _downloadProductFiles( # file was downloaded (200), or skipped before downloading (777) fileList.append(dpf.getInfo()) index += 1 - dpf = _DataProductFile(runId, str(index), baseUrl, token) + dpf = _DataProductFile(runId, str(index), baseUrl, token, self._config("verbosity")) elif status != 202 or (fileCount > 0 and index >= fileCount): # no more files to download @@ -219,7 +216,7 @@ def _downloadProductFiles( # get metadata if required if getMetadata: - dpf = _DataProductFile(runId, "meta", baseUrl, token) + dpf = _DataProductFile(runId, "meta", baseUrl, token, self._config("verbosity")) try: status = dpf.download( timeout, @@ -248,8 +245,8 @@ def _infoForProductFiles(self, dpRunId: int, fileCount: int, getMetadata: bool): Returned rows will have the same structure as those returned by _DataProductFile.getInfo(). """ - print( - f"\nObtaining download information for data product files with runId {dpRunId}..." # noqa: E501 + self._log.info( + f"Obtaining download information for data product files with runId {dpRunId}..." ) # If we don't know the fileCount, get it from the server (takes longer) @@ -268,6 +265,7 @@ def _infoForProductFiles(self, dpRunId: int, fileCount: int, getMetadata: bool): index=str(index), baseUrl=self._config("baseUrl"), token=self._config("token"), + verbosity=self._config("verbosity"), ) dpf.setComplete() fileList.append(dpf.getInfo()) @@ -304,7 +302,7 @@ def _countFilesInProduct(self, runId: int): filters["index"] += 1 n += 1 - print(f" {n} files available for download") + self._log.info(f" {n} files available for download") return n def _printProductRequest(self, response): @@ -315,19 +313,19 @@ def _printProductRequest(self, response): product source (archive or generated on the fly). """ isGenerated = "estimatedFileSize" in response - print(f"Request Id: {response['dpRequestId']}") + self._log.info(f"Request Id: {response['dpRequestId']}") if isGenerated: size = response["estimatedFileSize"] # API returns it as a formatted string - print(f"Estimated File Size: {size}") + self._log.info(f"Estimated File Size: {size}") if "estimatedProcessingTime" in response: - print( + self._log.info( f"Estimated Processing Time: {response['estimatedProcessingTime']}" ) else: size = _formatSize(response["fileSize"]) - print(f"File Size: {size}") - print("Data product is ready for download.") + self._log.info(f"File Size: {size}") + self._log.info("Data product is ready for download.") def _estimatePollPeriod(self, response): """ @@ -373,7 +371,7 @@ def _printProductOrderStats(self, fileList: list, runInfo: dict): # Print run time runTime = timedelta(seconds=runInfo["runTime"]) - print(f"Total run time: {humanize.naturaldelta(runTime)}") + self._log.info(f"Total run time: {humanize.naturaldelta(runTime)}") if downloadCount > 0: # Print download time @@ -381,13 +379,13 @@ def _printProductOrderStats(self, fileList: list, runInfo: dict): txtDownTime = f"{downloadTime:.3f} seconds" else: txtDownTime = humanize.naturaldelta(downloadTime) - print(f"Total download Time: {txtDownTime}") + self._log.info(f"Total download Time: {txtDownTime}") # Print size and count of files natural_size = humanize.naturalsize(size, binary=True) - print(f"{downloadCount} files ({natural_size}) downloaded") + self._log.info(f"{downloadCount} files ({natural_size}) downloaded") else: - print("No files downloaded.") + self._log.info("No files downloaded.") def _formatResult(self, fileList: list, runInfo: dict): size = 0 diff --git a/src/onc/modules/_OncDiscovery.py b/src/onc/modules/_OncDiscovery.py index c6471c1..eb0fd74 100644 --- a/src/onc/modules/_OncDiscovery.py +++ b/src/onc/modules/_OncDiscovery.py @@ -7,8 +7,8 @@ class _OncDiscovery(_OncService): locations, deployments, devices, deviceCategories, properties, dataProducts """ - def __init__(self, parent: object, verbosity: str, redact_token: str, raise_http_errors: bool): - super().__init__(parent, verbosity, redact_token, raise_http_errors) + def __init__(self, parent: object): + super().__init__(parent) def _discoveryRequest(self, filters: dict, service: str): diff --git a/src/onc/modules/_OncRealTime.py b/src/onc/modules/_OncRealTime.py index f8c78ba..7ce93ec 100644 --- a/src/onc/modules/_OncRealTime.py +++ b/src/onc/modules/_OncRealTime.py @@ -9,8 +9,8 @@ class _OncRealTime(_OncService): Near real-time services methods """ - def __init__(self, config: dict, verbosity: str, redact_token: str, raise_http_errors: bool): - super().__init__(config, verbosity, redact_token, raise_http_errors) + def __init__(self, parent: object): + super().__init__(parent) def getScalardataByLocation(self, filters: dict, allPages: bool): """ @@ -90,7 +90,7 @@ def _getDirectAllPages(self, filters: dict, service: str, allPages: bool) -> Any filters["sensorCategoryCodes"] = ",".join(filters["sensorCategoryCodes"]) if allPages: - mp = _MultiPage(self, self.verbosity, self.raise_http_errors) + mp = _MultiPage(self) result = mp.getAllPages(service, url, filters) else: result = self._doRequest(url, filters) diff --git a/src/onc/modules/_OncService.py b/src/onc/modules/_OncService.py index c8a251c..4f156a0 100644 --- a/src/onc/modules/_OncService.py +++ b/src/onc/modules/_OncService.py @@ -1,5 +1,4 @@ import logging -import pprint import weakref from time import time from urllib import parse @@ -14,7 +13,7 @@ RESPONSE_TIME_MSG, RESPONSE_MSG) -logging.basicConfig(format="%(levelname)s: %(message)s") + class _OncService: @@ -22,16 +21,9 @@ class _OncService: Provides common configuration and functionality to Onc service classes (children) """ - def __init__(self, parent: object, - verbosity: str, - redact_token: bool, - raise_http_errors: bool): + def __init__(self, parent: object): self.parent = weakref.ref(parent) - self.redact_token = redact_token - self.raise_http_errors = raise_http_errors - self.verbosity = verbosity - - self.__log = setup_logger('onc-service', level = verbosity) + self._log = setup_logger('onc.service', level=self._config('verbosity')) def _doRequest(self, url: str, filters: dict | None = None, getTime: bool = False): """ @@ -60,7 +52,7 @@ def _doRequest(self, url: str, filters: dict | None = None, getTime: bool = Fals response = requests.get(url, filters, timeout=timeout) - if self.redact_token is True: + if self._config("redact_token"): try: response_url = scrub_token(response.url) except: @@ -69,48 +61,37 @@ def _doRequest(self, url: str, filters: dict | None = None, getTime: bool = Fals response_url = response.url # Log the url the user submitted. - self.__log.info(REQ_MSG.format(response_url)) + self._log.debug(REQ_MSG.format(response_url)) # Display the time it took for ONC to respond in seconds. # The requests.Response.elapsed value is a datetime.timedelta object. responseTime = round(response.elapsed.total_seconds(),3) # To milliseconds. - self.__log.debug(RESPONSE_TIME_MSG.format(responseTime)) + self._log.debug(RESPONSE_TIME_MSG.format(responseTime)) json_response = response.json() - if response.status_code == requests.codes.ok: - self.__log.info(RESPONSE_MSG.format("OK", response.status_code)) - if getTime is True: + # Log warning messages if json_response["messages"] is not an empty list + if "messages" in json_response and json_response["messages"]: + for message in json_response["messages"]: + self._log.warning(f"* {message}") + + if response.status_code in (requests.codes.ok, requests.codes.accepted): + self._log.debug(RESPONSE_MSG.format(response.reason, response.status_code)) + if getTime: return json_response, responseTime - else: - return json_response + return json_response else: - if response.status_code == requests.codes.not_found: - self.__log.error(RESPONSE_MSG.format("Not Found", - response.status_code)) - elif response.status_code == requests.codes.bad: - self.__log.error(RESPONSE_MSG.format("Bad Request", - response.status_code)) - elif response.status_code == requests.codes.unauthorized: - self.__log.error(RESPONSE_MSG.format("Unauthorized Request", - response.status_code)) - elif response.status_code == requests.codes.internal_server_error: - self.__log.error(RESPONSE_MSG.format("Internal Server Error", - response.status_code)) - else: - self.__log.error(RESPONSE_MSG.format('Error',response.status_code)) - - self.__log.error(build_error_message(response, - self.redact_token)) - - if self.raise_http_errors is True: + self._log.error(RESPONSE_MSG.format(response.reason, response.status_code)) + + self._log.error(build_error_message(response, + self._config("redact_token"))) + + if self._config("raise_http_errors"): response.raise_for_status() - else: - if getTime is True: - return response, responseTime - else: - return response + if getTime: + return response, responseTime + return response def _serviceUrl(self, service: str): diff --git a/src/onc/modules/_PollLog.py b/src/onc/modules/_PollLog.py index 1d32620..2de876f 100644 --- a/src/onc/modules/_PollLog.py +++ b/src/onc/modules/_PollLog.py @@ -1,24 +1,26 @@ +from onc.modules._Messages import setup_logger + class _PollLog: """ A helper for DataProductFile Keeps track of the messages printed in a single product download process """ - def __init__(self, showInfo: bool): + def __init__(self, verbosity: str): """ - @param showInfo same as in parent ONC object + @param verbosity standard logging level string """ self._messages = [] # unique messages returned during the product order self._runStart = 0.0 # {float} timestamp (seconds) self._runEnd = 0.0 - self._showInfo = showInfo # flag for writing console messages - self._doPrintFileCount = True - self._lastPrintedDot = False # True after printing a dot (.) without a newline + self._verbosity = verbosity + self._log = setup_logger('onc.poll', level=verbosity) + self._doLogFileCount = True def logMessage(self, response): """ Adds a message to the messages list if it's new - Prints message to console, or "." if it repeats itself + Logs message to logger """ # Detect if the response comes from a "run" or "download" method origin = "download" @@ -35,33 +37,18 @@ def logMessage(self, response): # Detect and print change in the file count if origin == "run": fileCount = response[0]["fileCount"] - if self._doPrintFileCount and fileCount > 0: - self.printInfo( - f"\n {fileCount} files generated for this data product", - True, - ) - self._doPrintFileCount = False + if self._doLogFileCount and fileCount > 0: + self._log.info(f" {fileCount} files generated for this data product") + self._doLogFileCount = False self._messages.append(msg) - self.printInfo("\n " + msg, sameLine=True) + self._log.info(" " + msg) else: - self.printInfo(".", sameLine=True) - - def printInfo(self, msg: str, sameLine: bool = False): - """ - Conditional printing helper - """ - self._lastPrintedDot = msg == "." - - if self._showInfo: - if sameLine: - print(msg, end="", flush=True) - else: - print(msg) + # For repeating messages, log at DEBUG level to avoid cluttering INFO logs + self._log.debug(msg) def printNewLine(self): """ - Prints a line break only if the last message printed was a dot (.) + Deprecated. Newlines are handled by the logger. """ - if self._lastPrintedDot: - print("") + pass diff --git a/src/onc/onc.py b/src/onc/onc.py index 328abb2..32695b2 100644 --- a/src/onc/onc.py +++ b/src/onc/onc.py @@ -1,5 +1,6 @@ import datetime import json +import logging import os import re from pathlib import Path @@ -28,14 +29,12 @@ class ONC: - True: Use the production server. - False: Use the internal ONC test server (reserved for ONC staff IP addresses). - showInfo : boolean, default False - Whether verbose script messages are displayed, such as request url and processing time information. - - - True: Print all information and debug messages (intended for debugging). - - False: Only print information messages. - showWarning : boolean, default True - Whether warning messages are displayed. Some web services have "messages" key in the response JSON - to indicate that something might need attention, like using a default value for a missing parameter. + verbosity : str, default "INFO" + The logging verbosity level. Supported values: "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL". + showInfo : boolean [Deprecated] + Use verbosity="DEBUG" or "INFO" instead. + showWarning : boolean [Deprecated] + Use verbosity="WARNING" instead. outPath : str | Path, default "output" The directory that files are saved to (relative to the current directory) when downloading files. The directory will be created if it does not exist during the download. @@ -47,7 +46,7 @@ class ONC: >>> from onc import ONC >>> onc = ONC() # Only if you set the env variable "ONC_TOKEN" # doctest: +SKIP >>> onc = ONC("YOUR_TOKEN_HERE") # doctest: +SKIP - >>> onc = ONC("YOUR_TOKEN_HERE", showInfo=True, outPath="onc-files") # doctest: +SKIP + >>> onc = ONC("YOUR_TOKEN_HERE", verbosity="DEBUG", outPath="onc-files") # doctest: +SKIP """ # noqa: E501 def __init__( @@ -59,7 +58,21 @@ def __init__( redact_token: bool = False, timeout: int = 60, production: bool = True, + **kwargs, ): + if "showInfo" in kwargs or "showWarning" in kwargs: + import warnings + warnings.warn( + "showInfo and showWarning are deprecated. Use verbosity instead.", + DeprecationWarning, + stacklevel=2, + ) + # If verbosity wasn't explicitly changed from default, use showInfo/showWarning to set it + if verbosity == "INFO": + if kwargs.get("showInfo") is True: + verbosity = "DEBUG" + if kwargs.get("showWarning") is False: + verbosity = "ERROR" if token is None or token == "": token = os.environ.get("ONC_TOKEN") if token is None or token == "": @@ -68,17 +81,10 @@ def __init__( "or set it as the environment variable 'ONC_TOKEN'." ) - self.verbosity = verbosity.upper() + self.verbosity = verbosity self.redact_token = redact_token self.raise_http_errors = raise_http_errors - self.__log = setup_logger('onc-client', self.verbosity) - - if self.verbosity in ['INFO', 'DEBUG']: - self.showInfo = True - self.showWarning = True - elif self.verbosity in ['WARNING', 'ERROR']: - self.showInfo = False - self.showWarning = True + self._log = setup_logger('onc', self.verbosity) self.token = re.sub(r"[^a-zA-Z0-9\-]+", "", token) @@ -87,12 +93,12 @@ def __init__( self.outPath = outPath # Create service objects - self.discovery = _OncDiscovery(self, verbosity, redact_token,raise_http_errors) - self.delivery = _OncDelivery(self, verbosity, redact_token,raise_http_errors) - self.realTime = _OncRealTime(self, verbosity, redact_token,raise_http_errors) - self.archive = _OncArchive(self, verbosity, redact_token,raise_http_errors) + self.discovery = _OncDiscovery(self) + self.delivery = _OncDelivery(self) + self.realTime = _OncRealTime(self) + self.archive = _OncArchive(self) - self.__log.debug("Initialized ONC module.") + self._log.debug("Initialized ONC module.") @property def outPath(self) -> Path: @@ -123,6 +129,28 @@ def production(self, is_production: bool) -> None: else: self.baseUrl = "https://qa.oceannetworks.ca/" + @property + def verbosity(self) -> str: + """ + Return the current verbosity level. + """ + return self._verbosity + + @verbosity.setter + def verbosity(self, level: str) -> None: + """ + Set the logging verbosity level for the ONC instance and all its services. + + Supported values: "DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL". + """ + self._verbosity = level.upper() + setup_logger("onc", self._verbosity) + + # Update all existing loggers that start with 'onc.' + for name in logging.root.manager.loggerDict: + if name.startswith("onc."): + setup_logger(name, self._verbosity) + def print(self, obj, filename: str = "") -> None: """ Pretty print a collection to the console or a file. @@ -154,6 +182,34 @@ def print(self, obj, filename: str = "") -> None: with open(filePath, "w+") as file: file.write(text) + @property + def showInfo(self) -> bool: + """ + [Deprecated] Return whether the logger level is DEBUG. + """ + return self._log.getEffectiveLevel() <= logging.DEBUG + + @showInfo.setter + def showInfo(self, value: bool) -> None: + """ + [Deprecated] Set the logger level based on the boolean value. + """ + self.verbosity = "DEBUG" if value else "INFO" + + @property + def showWarning(self) -> bool: + """ + [Deprecated] Return whether the logger level is WARNING or lower. + """ + return self._log.getEffectiveLevel() <= logging.WARNING + + @showWarning.setter + def showWarning(self, value: bool) -> None: + """ + [Deprecated] Set the logger level based on the boolean value. + """ + self.verbosity = "WARNING" if value else "ERROR" + def formatUtc(self, dateString: str = "now") -> str: """ Format the provided date string as an ISO8601 UTC date string. From 4f878eff5be9fc6bb4c796b77725dfa723add45e Mon Sep 17 00:00:00 2001 From: Kan Fu Date: Wed, 22 Apr 2026 11:49:15 -0700 Subject: [PATCH 3/4] refactor: standardize test error handling by injecting err_400 and err_404 fixtures --- pyproject.toml | 3 +++ tests/archive_file/test_archivefile_device.py | 8 ++++---- .../archive_file/test_archivefile_download.py | 8 ++++---- .../archive_file/test_archivefile_location.py | 12 +++++------ tests/conftest.py | 11 +++++++++- .../test_data_product_delivery_cancel.py | 4 ++-- .../test_data_product_delivery_download.py | 4 ++-- .../test_data_product_delivery_order.py | 7 +++++-- .../test_data_product_delivery_restart.py | 4 ++-- .../test_data_product_delivery_run.py | 4 ++-- .../test_data_product_delivery_status.py | 4 ++-- .../data_product_delivery/test_integration.py | 2 ++ .../test_data_availability.py | 4 ++-- .../test_data_products.py | 12 +++++------ .../discover_deployments/test_deployments.py | 20 +++++++++---------- .../test_device_categories.py | 12 +++++------ tests/discover_devices/test_devices.py | 20 +++++++++---------- tests/discover_locations/test_locations.py | 20 +++++++++---------- .../discover_locations/test_locations_tree.py | 20 +++++++++---------- tests/discover_properties/test_properties.py | 12 +++++------ tests/raw_data/test_rawdata_device.py | 8 ++++---- tests/raw_data/test_rawdata_location.py | 12 +++++------ tests/scalar_data/test_scalardata_device.py | 8 ++++---- tests/scalar_data/test_scalardata_location.py | 12 +++++------ 24 files changed, 124 insertions(+), 107 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 0c4e31e..6a9e0f7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -85,3 +85,6 @@ testpaths = [ addopts = [ "--import-mode=importlib", ] +markers = [ + "slow: marks tests as slow (deselect with '-m \"not slow\"')", +] diff --git a/tests/archive_file/test_archivefile_device.py b/tests/archive_file/test_archivefile_device.py index f6ebdc0..a589f9b 100644 --- a/tests/archive_file/test_archivefile_device.py +++ b/tests/archive_file/test_archivefile_device.py @@ -20,9 +20,9 @@ def params_multiple_pages(params): return params | {"rowLimit": 2} -def test_invalid_param_value(requester, params): +def test_invalid_param_value(requester, params, err_400): params_invalid_param_value = params | {"deviceCode": "XYZ123"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getArchivefile(params_invalid_param_value) @@ -32,9 +32,9 @@ def test_invalid_params_missing_required(requester, params): requester.getArchivefile(params) -def test_invalid_param_name(requester, params): +def test_invalid_param_name(requester, params, err_400): params_invalid_param_name = params | {"deviceCodes": "BPR-Folger-59"} - with pytest.raises(requests.HTTPError, match=r"API Error 129"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getArchivefile(params_invalid_param_name) diff --git a/tests/archive_file/test_archivefile_download.py b/tests/archive_file/test_archivefile_download.py index 8ae6c9b..ef0900b 100644 --- a/tests/archive_file/test_archivefile_download.py +++ b/tests/archive_file/test_archivefile_download.py @@ -4,13 +4,13 @@ import requests -def test_invalid_param_value(requester): - with pytest.raises(requests.HTTPError, match=r"API Error 96"): +def test_invalid_param_value(requester, err_400): + with pytest.raises(requests.HTTPError, match=err_400): requester.downloadArchivefile("FAKEFILE.XYZ") -def test_invalid_params_missing_required(requester): - with pytest.raises(requests.HTTPError, match=r"API Error 128"): +def test_invalid_params_missing_required(requester, err_400): + with pytest.raises(requests.HTTPError, match=err_400): requester.downloadArchivefile() diff --git a/tests/archive_file/test_archivefile_location.py b/tests/archive_file/test_archivefile_location.py index 89fb1b6..9900b21 100644 --- a/tests/archive_file/test_archivefile_location.py +++ b/tests/archive_file/test_archivefile_location.py @@ -2,9 +2,9 @@ import requests -def test_invalid_param_value(requester, params_location): +def test_invalid_param_value(requester, params_location, err_400): params_invalid_param_value = params_location | {"locationCode": "XYZ123"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getArchivefile(params_invalid_param_value) @@ -14,18 +14,18 @@ def test_invalid_params_missing_required(requester, params_location): requester.getArchivefile(params_location) -def test_invalid_param_name(requester, params_location): +def test_invalid_param_name(requester, params_location, err_400): params_invalid_param_name = params_location | {"locationCodes": "NCBC"} - with pytest.raises(requests.HTTPError, match=r"API Error 129"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getArchivefile(params_invalid_param_name) -def test_no_data(requester, params_location): +def test_no_data(requester, params_location, err_400): params_no_data = params_location | { "dateFrom": "2000-01-01", "dateTo": "2000-01-02", } - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getArchivefile(params_no_data) diff --git a/tests/conftest.py b/tests/conftest.py index 12d434d..cc87c02 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,6 @@ import os from pathlib import Path - +import requests import pytest from dotenv import load_dotenv from onc import ONC @@ -18,6 +18,15 @@ def pytest_configure(): def requester(tmp_path) -> ONC: return ONC(production=is_prod, outPath=tmp_path) +@pytest.fixture +def err_400(): + return f"{requests.codes.bad} Client Error" + + +@pytest.fixture +def err_404(): + return f"{requests.codes.not_found} Client Error" + @pytest.fixture(scope="session") def util(): diff --git a/tests/data_product_delivery/test_data_product_delivery_cancel.py b/tests/data_product_delivery/test_data_product_delivery_cancel.py index ab347b7..f68b19e 100644 --- a/tests/data_product_delivery/test_data_product_delivery_cancel.py +++ b/tests/data_product_delivery/test_data_product_delivery_cancel.py @@ -2,6 +2,6 @@ import requests -def test_invalid_request_id(requester): - with pytest.raises(requests.HTTPError, match=r"API Error 127"): +def test_invalid_request_id(requester, err_400): + with pytest.raises(requests.HTTPError, match=err_400): requester.cancelDataProduct(1234567890) diff --git a/tests/data_product_delivery/test_data_product_delivery_download.py b/tests/data_product_delivery/test_data_product_delivery_download.py index faacce4..14c000a 100644 --- a/tests/data_product_delivery/test_data_product_delivery_download.py +++ b/tests/data_product_delivery/test_data_product_delivery_download.py @@ -2,6 +2,6 @@ import requests -def test_invalid_run_id(requester): - with pytest.raises(requests.HTTPError, match=r"API Error 127"): +def test_invalid_run_id(requester, err_400): + with pytest.raises(requests.HTTPError, match=err_400): requester.downloadDataProduct(1234567890) diff --git a/tests/data_product_delivery/test_data_product_delivery_order.py b/tests/data_product_delivery/test_data_product_delivery_order.py index 7b16df2..06a6524 100644 --- a/tests/data_product_delivery/test_data_product_delivery_order.py +++ b/tests/data_product_delivery/test_data_product_delivery_order.py @@ -2,12 +2,13 @@ import requests -def test_invalid_param_value(requester, params): +def test_invalid_param_value(requester, params, err_400): params_invalid_param_value = params | {"dataProductCode": "XYZ123"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.orderDataProduct(params_invalid_param_value) +@pytest.mark.slow def test_valid_default(requester, params, expected_keys_download_results, util): data = requester.orderDataProduct(params) @@ -30,6 +31,7 @@ def test_valid_default(requester, params, expected_keys_download_results, util): ) +@pytest.mark.slow def test_valid_no_metadata(requester, params, expected_keys_download_results, util): data = requester.orderDataProduct(params, includeMetadataFile=False) @@ -49,6 +51,7 @@ def test_valid_no_metadata(requester, params, expected_keys_download_results, ut ) +@pytest.mark.slow def test_valid_results_only(requester, params, expected_keys_download_results, util): data = requester.orderDataProduct(params, downloadResultsOnly=True) diff --git a/tests/data_product_delivery/test_data_product_delivery_restart.py b/tests/data_product_delivery/test_data_product_delivery_restart.py index 4cf1fe5..672ad59 100644 --- a/tests/data_product_delivery/test_data_product_delivery_restart.py +++ b/tests/data_product_delivery/test_data_product_delivery_restart.py @@ -2,6 +2,6 @@ import requests -def test_invalid_request_id(requester): - with pytest.raises(requests.HTTPError, match=r"API Error 127"): +def test_invalid_request_id(requester, err_400): + with pytest.raises(requests.HTTPError, match=err_400): requester.restartDataProduct(1234567890) diff --git a/tests/data_product_delivery/test_data_product_delivery_run.py b/tests/data_product_delivery/test_data_product_delivery_run.py index 76463c1..0e3dcdc 100644 --- a/tests/data_product_delivery/test_data_product_delivery_run.py +++ b/tests/data_product_delivery/test_data_product_delivery_run.py @@ -2,6 +2,6 @@ import requests -def test_invalid_request_id(requester): - with pytest.raises(requests.HTTPError, match=r"API Error 127"): +def test_invalid_request_id(requester, err_400): + with pytest.raises(requests.HTTPError, match=err_400): requester.runDataProduct(1234567890) diff --git a/tests/data_product_delivery/test_data_product_delivery_status.py b/tests/data_product_delivery/test_data_product_delivery_status.py index a5eb452..4a48aec 100644 --- a/tests/data_product_delivery/test_data_product_delivery_status.py +++ b/tests/data_product_delivery/test_data_product_delivery_status.py @@ -2,6 +2,6 @@ import requests -def test_invalid_request_id(requester): - with pytest.raises(requests.HTTPError, match=r"API Error 127"): +def test_invalid_request_id(requester, err_400): + with pytest.raises(requests.HTTPError, match=err_400): requester.checkDataProduct(1234567890) diff --git a/tests/data_product_delivery/test_integration.py b/tests/data_product_delivery/test_integration.py index 39014c7..1c67ee5 100644 --- a/tests/data_product_delivery/test_integration.py +++ b/tests/data_product_delivery/test_integration.py @@ -1,6 +1,7 @@ import pytest +@pytest.mark.slow def test_valid_manual(requester, params, expected_keys_download_results, util): """ Test request -> status -> run -> download -> status. @@ -35,6 +36,7 @@ def test_valid_manual(requester, params, expected_keys_download_results, util): assert data_status_after_download["searchHdrStatus"] == "COMPLETED" +@pytest.mark.slow def test_valid_cancel_restart(requester, params, expected_keys_download_results, util): """ Test request -> run -> cancel -> download (fail) -> restart -> download. diff --git a/tests/discover_data_availability/test_data_availability.py b/tests/discover_data_availability/test_data_availability.py index 1a4a201..a748707 100644 --- a/tests/discover_data_availability/test_data_availability.py +++ b/tests/discover_data_availability/test_data_availability.py @@ -12,9 +12,9 @@ def params(): } -def test_invalid_param_value(requester, params): +def test_invalid_param_value(requester, params, err_400): params_invalid_param_value = params | {"locationCode": "INVALID"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getDataAvailability(params_invalid_param_value) diff --git a/tests/discover_data_products/test_data_products.py b/tests/discover_data_products/test_data_products.py index 6d149bc..8d75281 100644 --- a/tests/discover_data_products/test_data_products.py +++ b/tests/discover_data_products/test_data_products.py @@ -2,25 +2,25 @@ import requests -def test_invalid_param_value(requester): +def test_invalid_param_value(requester, err_400): params_invalid_param_value = {"dataProductCode": "XYZ123"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getDataProducts(params_invalid_param_value) -def test_invalid_param_name(requester): +def test_invalid_param_name(requester, err_400): params_invalid_param_name = {"dataProductCodes": "HSD"} - with pytest.raises(requests.HTTPError, match=r"API Error 129"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getDataProducts(params_invalid_param_name) -def test_no_data(requester): +def test_no_data(requester, err_404): params_no_data = { "dataProductCode": "HSD", "extension": "txt", } - with pytest.raises(requests.HTTPError, match=r"404 Client Error"): + with pytest.raises(requests.HTTPError, match=err_404): requester.getDataProducts(params_no_data) diff --git a/tests/discover_deployments/test_deployments.py b/tests/discover_deployments/test_deployments.py index b984978..76b3b30 100644 --- a/tests/discover_deployments/test_deployments.py +++ b/tests/discover_deployments/test_deployments.py @@ -2,46 +2,46 @@ import requests -def test_invalid_time_range_greater_start_time(requester): +def test_invalid_time_range_greater_start_time(requester, err_400): params_invalid_time_range_greater_start_time = { "locationCode": "BACAX", "deviceCategoryCode": "CTD", "dateFrom": "2020-01-01", "dateTo": "2019-01-01", } - with pytest.raises(requests.HTTPError, match=r"API Error 23"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getDeployments(params_invalid_time_range_greater_start_time) -def test_invalid_time_range_future_start_time(requester): +def test_invalid_time_range_future_start_time(requester, err_400): params_invalid_time_range_future_start_time = { "locationCode": "BACAX", "deviceCategoryCode": "CTD", "dateFrom": "2050-01-01", } - with pytest.raises(requests.HTTPError, match=r"API Error 25"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getDeployments(params_invalid_time_range_future_start_time) -def test_invalid_param_value(requester): +def test_invalid_param_value(requester, err_400): params_invalid_param_value = {"locationCode": "XYZ123", "deviceCategoryCode": "CTD"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getDeployments(params_invalid_param_value) -def test_invalid_param_name(requester): +def test_invalid_param_name(requester, err_400): params_invalid_param_name = {"locationCodes": "BACAX", "deviceCategoryCode": "CTD"} - with pytest.raises(requests.HTTPError, match=r"API Error 129"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getDeployments(params_invalid_param_name) -def test_no_data(requester): +def test_no_data(requester, err_400): params_no_data = { "locationCode": "BACAX", "deviceCategoryCode": "CTD", "dateTo": "1900-01-01", } - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getDeployments(params_no_data) diff --git a/tests/discover_device_categories/test_device_categories.py b/tests/discover_device_categories/test_device_categories.py index 45a2a0f..3a39445 100644 --- a/tests/discover_device_categories/test_device_categories.py +++ b/tests/discover_device_categories/test_device_categories.py @@ -2,25 +2,25 @@ import requests -def test_invalid_param_value(requester): +def test_invalid_param_value(requester, err_400): params_invalid_param_value = {"deviceCategoryCode": "XYZ123"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getDeviceCategories(params_invalid_param_value) -def test_invalid_param_name(requester): +def test_invalid_param_name(requester, err_400): params_invalid_param_name = {"deviceCategoryCodes": "CTD"} - with pytest.raises(requests.HTTPError, match=r"API Error 129"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getDeviceCategories(params_invalid_param_name) -def test_no_data(requester): +def test_no_data(requester, err_404): params_no_data = { "deviceCategoryCode": "CTD", "deviceCategoryName": "Conductivity", "description": "TemperatureXXX", } - with pytest.raises(requests.HTTPError, match=r"404 Client Error"): + with pytest.raises(requests.HTTPError, match=err_404): requester.getDeviceCategories(params_no_data) diff --git a/tests/discover_devices/test_devices.py b/tests/discover_devices/test_devices.py index 96069aa..1ffa135 100644 --- a/tests/discover_devices/test_devices.py +++ b/tests/discover_devices/test_devices.py @@ -2,40 +2,40 @@ import requests -def test_invalid_time_range_greater_start_time(requester): +def test_invalid_time_range_greater_start_time(requester, err_400): params_invalid_time_range_greater_start_time = { "deviceCode": "BPR-Folger-59", "dateFrom": "2020-01-01", "dateTo": "2019-01-01", } - with pytest.raises(requests.HTTPError, match=r"API Error 23"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getDevices(params_invalid_time_range_greater_start_time) -def test_invalid_time_range_future_start_time(requester): +def test_invalid_time_range_future_start_time(requester, err_400): params_invalid_time_range_future_start_time = { "deviceCode": "BPR-Folger-59", "dateFrom": "2050-01-01", } - with pytest.raises(requests.HTTPError, match=r"API Error 25"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getDevices(params_invalid_time_range_future_start_time) -def test_invalid_param_value(requester): +def test_invalid_param_value(requester, err_400): params_invalid_param_value = {"deviceCode": "XYZ123"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getDevices(params_invalid_param_value) -def test_invalid_param_name(requester): +def test_invalid_param_name(requester, err_400): params_invalid_param_name = {"deviceCodes": "BPR-Folger-59"} - with pytest.raises(requests.HTTPError, match=r"API Error 129"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getDevices(params_invalid_param_name) -def test_no_data(requester): +def test_no_data(requester, err_404): params_no_data = {"deviceCode": "BPR-Folger-59", "dateTo": "1900-01-01"} - with pytest.raises(requests.HTTPError, match=r"404 Client Error"): + with pytest.raises(requests.HTTPError, match=err_404): requester.getDevices(params_no_data) diff --git a/tests/discover_locations/test_locations.py b/tests/discover_locations/test_locations.py index 2160042..af60aa0 100644 --- a/tests/discover_locations/test_locations.py +++ b/tests/discover_locations/test_locations.py @@ -2,40 +2,40 @@ import requests -def test_invalid_time_range_greater_start_time(requester): +def test_invalid_time_range_greater_start_time(requester, err_400): params_invalid_time_range_greater_start_time = { "locationCode": "FGPD", "dateFrom": "2020-01-01", "dateTo": "2019-01-01", } - with pytest.raises(requests.HTTPError, match=r"API Error 23"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getLocations(params_invalid_time_range_greater_start_time) -def test_invalid_time_range_future_start_time(requester): +def test_invalid_time_range_future_start_time(requester, err_400): params_invalid_time_range_future_start_time = { "locationCode": "FGPD", "dateFrom": "2050-01-01", } - with pytest.raises(requests.HTTPError, match=r"API Error 25"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getLocations(params_invalid_time_range_future_start_time) -def test_invalid_param_value(requester): +def test_invalid_param_value(requester, err_400): params_invalid_param_value = {"locationCode": "XYZ123"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getLocations(params_invalid_param_value) -def test_invalid_param_name(requester): +def test_invalid_param_name(requester, err_400): params_invalid_param_name = {"locationCodes": "FGPD"} - with pytest.raises(requests.HTTPError, match=r"API Error 129"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getLocations(params_invalid_param_name) -def test_no_data(requester): +def test_no_data(requester, err_404): params_no_data = {"locationCode": "FGPD", "dateTo": "1900-01-01"} - with pytest.raises(requests.HTTPError, match=r"404 Client Error"): + with pytest.raises(requests.HTTPError, match=err_404): requester.getLocations(params_no_data) diff --git a/tests/discover_locations/test_locations_tree.py b/tests/discover_locations/test_locations_tree.py index e0eb47a..388c06a 100644 --- a/tests/discover_locations/test_locations_tree.py +++ b/tests/discover_locations/test_locations_tree.py @@ -2,40 +2,40 @@ import requests -def test_invalid_time_range_greater_start_time(requester): +def test_invalid_time_range_greater_start_time(requester, err_400): params_invalid_time_range_greater_start_time = { "locationCode": "ARCT", "dateFrom": "2020-01-01", "dateTo": "2019-01-01", } - with pytest.raises(requests.HTTPError, match=r"API Error 23"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getLocationsTree(params_invalid_time_range_greater_start_time) -def test_invalid_time_range_future_start_time(requester): +def test_invalid_time_range_future_start_time(requester, err_400): params_invalid_time_range_future_start_time = { "locationCode": "ARCT", "dateFrom": "2050-01-01", } - with pytest.raises(requests.HTTPError, match=r"API Error 25"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getLocationsTree(params_invalid_time_range_future_start_time) -def test_invalid_param_value(requester): +def test_invalid_param_value(requester, err_400): params_invalid_param_value = {"locationCode": "XYZ123"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getLocationsTree(params_invalid_param_value) -def test_invalid_param(requester): +def test_invalid_param(requester, err_400): params_invalid_param_name = {"locationCodes": "ARCT"} - with pytest.raises(requests.HTTPError, match=r"API Error 129"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getLocationsTree(params_invalid_param_name) -def test_no_data(requester): +def test_no_data(requester, err_404): params_no_data = {"locationCode": "ARCT", "dateTo": "1900-01-01"} - with pytest.raises(requests.HTTPError, match=r"404 Client Error"): + with pytest.raises(requests.HTTPError, match=err_404): requester.getLocationsTree(params_no_data) diff --git a/tests/discover_properties/test_properties.py b/tests/discover_properties/test_properties.py index a72d88b..6703601 100644 --- a/tests/discover_properties/test_properties.py +++ b/tests/discover_properties/test_properties.py @@ -2,25 +2,25 @@ import requests -def test_invalid_param_value(requester): +def test_invalid_param_value(requester, err_400): params_invalid_param_value = {"propertyCode": "XYZ123"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getProperties(params_invalid_param_value) -def test_invalid_param_name(requester): +def test_invalid_param_name(requester, err_400): params_invalid_param_name = {"propertyCodes": "conductivity"} - with pytest.raises(requests.HTTPError, match=r"API Error 129"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getProperties(params_invalid_param_name) -def test_no_data(requester): +def test_no_data(requester, err_404): params_no_data = { "propertyCode": "conductivity", "locationCode": "SAAN", } - with pytest.raises(requests.HTTPError, match=r"404 Client Error"): + with pytest.raises(requests.HTTPError, match=err_404): requester.getProperties(params_no_data) diff --git a/tests/raw_data/test_rawdata_device.py b/tests/raw_data/test_rawdata_device.py index 2e13959..6fe6585 100644 --- a/tests/raw_data/test_rawdata_device.py +++ b/tests/raw_data/test_rawdata_device.py @@ -18,15 +18,15 @@ def params_multiple_pages(params): return params | {"rowLimit": 25} -def test_invalid_param_value(requester, params): +def test_invalid_param_value(requester, params, err_400): params_invalid_param_value = params | {"deviceCode": "XYZ123"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getRawdata(params_invalid_param_value) -def test_invalid_param_name(requester, params): +def test_invalid_param_name(requester, params, err_400): params_invalid_param_name = params | {"deviceCodes": "BPR-Folger-59"} - with pytest.raises(requests.HTTPError, match=r"API Error 129"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getRawdata(params_invalid_param_name) diff --git a/tests/raw_data/test_rawdata_location.py b/tests/raw_data/test_rawdata_location.py index e9af23f..9889c45 100644 --- a/tests/raw_data/test_rawdata_location.py +++ b/tests/raw_data/test_rawdata_location.py @@ -21,21 +21,21 @@ def params_multiple_pages(params): return params | {"rowLimit": 25} -def test_invalid_param_value(requester, params): +def test_invalid_param_value(requester, params, err_400): params_invalid_param_value = params | {"locationCode": "XYZ123"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getRawdata(params_invalid_param_value) -def test_invalid_param_name(requester, params): +def test_invalid_param_name(requester, params, err_400): params_invalid_param_name = params | {"locationCodes": "NCBC"} - with pytest.raises(requests.HTTPError, match=r"API Error 129"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getRawdata(params_invalid_param_name) -def test_no_data(requester, params): +def test_no_data(requester, params, err_400): params_no_data = params | {"dateFrom": "2000-01-01", "dateTo": "2000-01-02"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getRawdata(params_no_data) diff --git a/tests/scalar_data/test_scalardata_device.py b/tests/scalar_data/test_scalardata_device.py index ffbb63e..c85d46a 100644 --- a/tests/scalar_data/test_scalardata_device.py +++ b/tests/scalar_data/test_scalardata_device.py @@ -8,15 +8,15 @@ def params_multiple_pages(params_device): return params_device | {"rowLimit": 25} -def test_invalid_param_value(requester, params_device): +def test_invalid_param_value(requester, params_device, err_400): params_invalid_param_value = params_device | {"deviceCode": "XYZ123"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getScalardata(params_invalid_param_value) -def test_invalid_param_name(requester, params_device): +def test_invalid_param_name(requester, params_device, err_400): params_invalid_param_name = params_device | {"deviceCodes": "BPR-Folger-59"} - with pytest.raises(requests.HTTPError, match=r"API Error 129"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getScalardata(params_invalid_param_name) diff --git a/tests/scalar_data/test_scalardata_location.py b/tests/scalar_data/test_scalardata_location.py index 19ed397..bb38752 100644 --- a/tests/scalar_data/test_scalardata_location.py +++ b/tests/scalar_data/test_scalardata_location.py @@ -8,24 +8,24 @@ def params_multiple_pages(params_location): return params_location | {"rowLimit": 25} -def test_invalid_param_value(requester, params_location): +def test_invalid_param_value(requester, params_location, err_400): params_invalid_param_value = params_location | {"locationCode": "XYZ123"} - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getScalardata(params_invalid_param_value) -def test_invalid_param_name(requester, params_location): +def test_invalid_param_name(requester, params_location, err_400): params_invalid_param_name = params_location | {"locationCodes": "NCBC"} - with pytest.raises(requests.HTTPError, match=r"API Error 129"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getScalardata(params_invalid_param_name) -def test_no_data(requester, params_location): +def test_no_data(requester, params_location, err_400): params_no_data = params_location | { "dateFrom": "2000-01-01", "dateTo": "2000-01-02", } - with pytest.raises(requests.HTTPError, match=r"API Error 127"): + with pytest.raises(requests.HTTPError, match=err_400): requester.getScalardata(params_no_data) From 19f99da777f804200b62add544757d79fc363a37 Mon Sep 17 00:00:00 2001 From: Kan Fu Date: Wed, 22 Apr 2026 12:19:47 -0700 Subject: [PATCH 4/4] refactor: apply style format and lint, use numpy docstring style --- pyproject.toml | 3 + scratch.ipynb | 2121 --------------------------- src/onc/modules/_DataProductFile.py | 16 +- src/onc/modules/_Messages.py | 89 +- src/onc/modules/_MultiPage.py | 68 +- src/onc/modules/_OncArchive.py | 4 +- src/onc/modules/_OncDelivery.py | 19 +- src/onc/modules/_OncDiscovery.py | 1 - src/onc/modules/_OncService.py | 35 +- src/onc/modules/_PollLog.py | 12 +- src/onc/modules/_util.py | 13 +- src/onc/onc.py | 5 +- src/onc/util/util.py | 11 +- tests/conftest.py | 4 +- 14 files changed, 181 insertions(+), 2220 deletions(-) delete mode 100644 scratch.ipynb diff --git a/pyproject.toml b/pyproject.toml index 6a9e0f7..08201db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,6 +77,9 @@ select = [ "B", # flake8-bugbear "SIM", # flake8-simplify ] +ignore = [ + "E501", +] [tool.pytest.ini_options] testpaths = [ diff --git a/scratch.ipynb b/scratch.ipynb deleted file mode 100644 index 06646c4..0000000 --- a/scratch.ipynb +++ /dev/null @@ -1,2121 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "id": "initial_id", - "metadata": { - "collapsed": true, - "ExecuteTime": { - "end_time": "2025-11-10T21:17:04.711252Z", - "start_time": "2025-11-10T21:17:04.581618Z" - } - }, - "source": [ - "from netrc import netrc\n", - "from onc import ONC\n", - "\n", - "_, __, token = netrc().authenticators('data.oceannetworks.ca')" - ], - "outputs": [], - "execution_count": 1 - }, - { - "metadata": {}, - "cell_type": "markdown", - "source": "## Initialization", - "id": "ef917e2e8133e385" - }, - { - "metadata": { - "ExecuteTime": { - "end_time": "2025-11-10T21:17:04.722824Z", - "start_time": "2025-11-10T21:17:04.717615Z" - } - }, - "cell_type": "code", - "source": [ - "message_level = 'DEBUG' # Prints all messages to console.\n", - "redact_token = True # Removes a users token from console messages if set to True.\n", - "raise_http_errors = False # Does not raise HTTPError and instead returns the full requests.Response object.\n", - "\n", - "onc = ONC(token = token, verbosity = message_level, redact_token=redact_token, raise_http_errors=raise_http_errors)" - ], - "id": "f762aa13502fee66", - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-11-10T21:17:04.720Z | onc-client | DEBUG | Initialized ONC module.\n" - ] - } - ], - "execution_count": 2 - }, - { - "metadata": { - "ExecuteTime": { - "end_time": "2025-11-10T21:00:57.054214Z", - "start_time": "2025-11-10T21:00:57.049825Z" - } - }, - "cell_type": "markdown", - "source": "## OK Discovery Example", - "id": "9b3a0033f2f05f6b" - }, - { - "metadata": { - "ExecuteTime": { - "end_time": "2025-11-10T21:17:05.247467Z", - "start_time": "2025-11-10T21:17:04.729847Z" - } - }, - "cell_type": "code", - "source": [ - "locs = onc.getLocationsTree({'locationCode': 'BCF'})\n", - "locs" - ], - "id": "5e6ed0b663a30da0", - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-11-10T21:17:05.232Z | onc-service | INFO | Requested: https://data.oceannetworks.ca/api/locations?locationCode=BCF&method=getTree&token=REDACTED\n", - "2025-11-10T21:17:05.232Z | onc-service | DEBUG | Response received in 0.5 seconds.\n", - "2025-11-10T21:17:05.232Z | onc-service | INFO | HTTP Response: OK (200)\n" - ] - }, - { - "data": { - "text/plain": [ - "[{'locationName': 'British Columbia Ferries',\n", - " 'children': [{'locationName': 'Campbell River - Quathiaski Cove Ferry Route',\n", - " 'children': None,\n", - " 'description': 'This ferry route goes between Campbell River, Vancouver Island, BC, and Quadra Island (Quathiaski Cove), BC.',\n", - " 'hasDeviceData': True,\n", - " 'locationCode': 'CRQC',\n", - " 'hasPropertyData': True},\n", - " {'locationName': 'Horseshoe Bay - Departure Bay Ferry Route',\n", - " 'children': [{'locationName': 'GNSS V104S GPS',\n", - " 'children': None,\n", - " 'description': None,\n", - " 'hasDeviceData': True,\n", - " 'locationCode': 'HBDB.N1',\n", - " 'hasPropertyData': True},\n", - " {'locationName': 'Thrane and Thrane SAILOR',\n", - " 'children': None,\n", - " 'description': None,\n", - " 'hasDeviceData': True,\n", - " 'locationCode': 'HBDB.N2',\n", - " 'hasPropertyData': False}],\n", - " 'description': 'This ferry route goes between Horseshoe Bay terminal in West Vancouver, BC and the Departure Bay terminal, Nanaimo BC.',\n", - " 'hasDeviceData': True,\n", - " 'locationCode': 'HBDB',\n", - " 'hasPropertyData': True},\n", - " {'locationName': 'Nanaimo Harbour - Descanso Bay Ferry Route',\n", - " 'children': None,\n", - " 'description': 'This ferry route goes between Nanaimo Harbour terminal in Nanaimo, BC and the Descanso Bay terminal on Gabriola Island, BC.',\n", - " 'hasDeviceData': True,\n", - " 'locationCode': 'NHDB',\n", - " 'hasPropertyData': True},\n", - " {'locationName': 'Port McNeill - Alert Bay - Sointula Ferry Route',\n", - " 'children': None,\n", - " 'description': 'This ferry route goes between Cormorant Island (Alert Bay), BC, Port McNeill, BC and Malcolm Island (Sointula), BC.',\n", - " 'hasDeviceData': True,\n", - " 'locationCode': 'PMABS',\n", - " 'hasPropertyData': True},\n", - " {'locationName': 'Tsawwassen - Duke Point Ferry Route',\n", - " 'children': [{'locationName': 'GNSS MV104S GPS',\n", - " 'children': None,\n", - " 'description': '',\n", - " 'hasDeviceData': True,\n", - " 'locationCode': 'TWDP.N1',\n", - " 'hasPropertyData': True},\n", - " {'locationName': 'Thrane and Thrane SAILOR',\n", - " 'children': None,\n", - " 'description': None,\n", - " 'hasDeviceData': True,\n", - " 'locationCode': 'TWDP.N2',\n", - " 'hasPropertyData': False}],\n", - " 'description': 'This ferry route goes between Tsawwassen terminal in Vancouver, BC and the Duke Point terminal, Nanaimo BC.',\n", - " 'hasDeviceData': True,\n", - " 'locationCode': 'TWDP',\n", - " 'hasPropertyData': True},\n", - " {'locationName': 'Tsawwassen - Swartz Bay Ferry Route',\n", - " 'children': [{'locationName': 'Fluorometer Secondary',\n", - " 'children': None,\n", - " 'description': '',\n", - " 'hasDeviceData': True,\n", - " 'locationCode': 'TWSB.F1',\n", - " 'hasPropertyData': False},\n", - " {'locationName': 'GNSS V104S GPS',\n", - " 'children': None,\n", - " 'description': None,\n", - " 'hasDeviceData': True,\n", - " 'locationCode': 'TWSB.N1',\n", - " 'hasPropertyData': False},\n", - " {'locationName': 'Thrane and Thrane SAILOR',\n", - " 'children': None,\n", - " 'description': None,\n", - " 'hasDeviceData': True,\n", - " 'locationCode': 'TWSB.N2',\n", - " 'hasPropertyData': False}],\n", - " 'description': 'This ferry route goes between Tsawwassen terminal in Vancouver, BC and the Swartz Bay terminal, Victoria BC.',\n", - " 'hasDeviceData': True,\n", - " 'locationCode': 'TWSB',\n", - " 'hasPropertyData': True}],\n", - " 'description': ' A few ferry vessels in the Salish Sea have been equipped with scientific instruments. Data is collected from routes between Vancouver Island and the mainland of British Columbia. Data is transmitted to Ocean Networks Canada shore stations via a wireless connection.',\n", - " 'hasDeviceData': False,\n", - " 'locationCode': 'BCF',\n", - " 'hasPropertyData': False}]" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "execution_count": 3 - }, - { - "metadata": {}, - "cell_type": "markdown", - "source": "## Not Found Discovery Example", - "id": "f983589d8201f9d3" - }, - { - "metadata": { - "ExecuteTime": { - "end_time": "2025-11-10T21:17:06.001885Z", - "start_time": "2025-11-10T21:17:05.290709Z" - } - }, - "cell_type": "code", - "source": [ - "locs = onc.getLocations({'locationCode': 'BCF',\n", - " 'dateFrom': '2000-01-01T00:00:00.000Z',\n", - " 'dateTo': '2000-01-01T23:59:59.999Z'})\n", - "locs.status_code # Output is a requests.Response object instead of a raised HTTPError" - ], - "id": "5e9b97e55e7b9419", - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-11-10T21:17:05.992Z | onc-service | INFO | Requested: https://data.oceannetworks.ca/api/locations?locationCode=BCF&dateFrom=2000-01-01T00%3A00%3A00.000Z&dateTo=2000-01-01T23%3A59%3A59.999Z&method=get&token=REDACTED\n", - "2025-11-10T21:17:05.992Z | onc-service | DEBUG | Response received in 0.695 seconds.\n", - "2025-11-10T21:17:05.992Z | onc-service | ERROR | HTTP Response: Not Found (404)\n", - "2025-11-10T21:17:05.992Z | onc-service | ERROR | No metadata/data found for the given query parameters. Check if parameter input is valid.\n" - ] - }, - { - "data": { - "text/plain": [ - "404" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "execution_count": 4 - }, - { - "metadata": {}, - "cell_type": "markdown", - "source": "## OK Data Example", - "id": "4c7c24a4da781d60" - }, - { - "metadata": { - "ExecuteTime": { - "end_time": "2025-11-10T21:18:00.702793Z", - "start_time": "2025-11-10T21:17:54.596415Z" - } - }, - "cell_type": "code", - "source": [ - "data = onc.getScalardata({'locationCode':'BACVP',\n", - " 'deviceCategoryCode':'CTD',\n", - " 'dateFrom': '2009-08-20T00:00:00.000Z',\n", - " 'dateTo': '2009-08-29T23:59:59.999Z',\n", - " 'rowLimit': 10}, allPages = True)\n", - "data['sensorData']" - ], - "id": "16e99050e0ab4c0a", - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-11-10T21:17:55.210Z | onc-service | INFO | Requested: https://data.oceannetworks.ca/api/scalardata?locationCode=BACVP&deviceCategoryCode=CTD&dateFrom=2009-08-20T00%3A00%3A00.000Z&dateTo=2009-08-29T23%3A59%3A59.999Z&rowLimit=10&method=getByLocation&token=REDACTED\n", - "2025-11-10T21:17:55.213Z | onc-service | DEBUG | Response received in 0.608 seconds.\n", - "2025-11-10T21:17:55.213Z | onc-service | INFO | HTTP Response: OK (200)\n", - "2025-11-10T21:17:55.217Z | onc-multi | INFO | The requested data quantity is greater than the supplied row limit and will be downloaded over multiple requests.\n", - "2025-11-10T21:17:55.219Z | onc-multi | DEBUG | Download time for page 1: 0.61 seconds\n", - "2025-11-10T21:17:55.222Z | onc-multi | INFO | Est. number of pages remaining for download: 114332\n", - "2025-11-10T21:17:55.222Z | onc-multi | INFO | Est. number of seconds to download remaining data: 19 hours\n", - "2025-11-10T21:17:55.223Z | onc-multi | DEBUG | Submitting request for page 2 (10 samples)...\n", - "2025-11-10T21:17:55.852Z | onc-service | INFO | Requested: https://data.oceannetworks.ca/api/scalardata?method=getByLocation&dateTo=2009-08-29T23%3A59%3A59.999Z&rowLimit=10&locationCode=BACVP&deviceCategoryCode=CTD&dateFrom=2009-08-27T00%3A53%3A32.289Z&token=REDACTED\n", - "2025-11-10T21:17:55.852Z | onc-service | DEBUG | Response received in 0.626 seconds.\n", - "2025-11-10T21:17:55.852Z | onc-service | INFO | HTTP Response: OK (200)\n", - "2025-11-10T21:17:55.858Z | onc-multi | DEBUG | Submitting request for page 3 (20 samples)...\n", - "2025-11-10T21:17:56.461Z | onc-service | INFO | Requested: https://data.oceannetworks.ca/api/scalardata?method=getByLocation&dateTo=2009-08-29T23%3A59%3A59.999Z&rowLimit=10&locationCode=BACVP&deviceCategoryCode=CTD&dateFrom=2009-08-27T00%3A54%3A33.083Z&token=REDACTED\n", - "2025-11-10T21:17:56.461Z | onc-service | DEBUG | Response received in 0.607 seconds.\n", - "2025-11-10T21:17:56.471Z | onc-service | INFO | HTTP Response: OK (200)\n", - "2025-11-10T21:17:56.471Z | onc-multi | DEBUG | Submitting request for page 4 (30 samples)...\n", - "2025-11-10T21:17:57.100Z | onc-service | INFO | Requested: https://data.oceannetworks.ca/api/scalardata?method=getByLocation&dateTo=2009-08-29T23%3A59%3A59.999Z&rowLimit=10&locationCode=BACVP&deviceCategoryCode=CTD&dateFrom=2009-08-27T00%3A55%3A33.075Z&token=REDACTED\n", - "2025-11-10T21:17:57.100Z | onc-service | DEBUG | Response received in 0.625 seconds.\n", - "2025-11-10T21:17:57.100Z | onc-service | INFO | HTTP Response: OK (200)\n", - "2025-11-10T21:17:57.107Z | onc-multi | DEBUG | Submitting request for page 5 (40 samples)...\n", - "2025-11-10T21:17:57.697Z | onc-service | INFO | Requested: https://data.oceannetworks.ca/api/scalardata?method=getByLocation&dateTo=2009-08-29T23%3A59%3A59.999Z&rowLimit=10&locationCode=BACVP&deviceCategoryCode=CTD&dateFrom=2009-08-27T00%3A56%3A32.975Z&token=REDACTED\n", - "2025-11-10T21:17:57.707Z | onc-service | DEBUG | Response received in 0.593 seconds.\n", - "2025-11-10T21:17:57.707Z | onc-service | INFO | HTTP Response: OK (200)\n", - "2025-11-10T21:17:57.707Z | onc-multi | DEBUG | Submitting request for page 6 (50 samples)...\n", - "2025-11-10T21:17:58.271Z | onc-service | INFO | Requested: https://data.oceannetworks.ca/api/scalardata?method=getByLocation&dateTo=2009-08-29T23%3A59%3A59.999Z&rowLimit=10&locationCode=BACVP&deviceCategoryCode=CTD&dateFrom=2009-08-27T00%3A57%3A32.881Z&token=REDACTED\n", - "2025-11-10T21:17:58.271Z | onc-service | DEBUG | Response received in 0.556 seconds.\n", - "2025-11-10T21:17:58.271Z | onc-service | INFO | HTTP Response: OK (200)\n", - "2025-11-10T21:17:58.271Z | onc-multi | DEBUG | Submitting request for page 7 (60 samples)...\n", - "2025-11-10T21:17:58.833Z | onc-service | INFO | Requested: https://data.oceannetworks.ca/api/scalardata?method=getByLocation&dateTo=2009-08-29T23%3A59%3A59.999Z&rowLimit=10&locationCode=BACVP&deviceCategoryCode=CTD&dateFrom=2009-08-27T00%3A58%3A32.891Z&token=REDACTED\n", - "2025-11-10T21:17:58.833Z | onc-service | DEBUG | Response received in 0.555 seconds.\n", - "2025-11-10T21:17:58.833Z | onc-service | INFO | HTTP Response: OK (200)\n", - "2025-11-10T21:17:58.833Z | onc-multi | DEBUG | Submitting request for page 8 (70 samples)...\n", - "2025-11-10T21:17:59.440Z | onc-service | INFO | Requested: https://data.oceannetworks.ca/api/scalardata?method=getByLocation&dateTo=2009-08-29T23%3A59%3A59.999Z&rowLimit=10&locationCode=BACVP&deviceCategoryCode=CTD&dateFrom=2009-08-27T00%3A59%3A32.886Z&token=REDACTED\n", - "2025-11-10T21:17:59.440Z | onc-service | DEBUG | Response received in 0.597 seconds.\n", - "2025-11-10T21:17:59.440Z | onc-service | INFO | HTTP Response: OK (200)\n", - "2025-11-10T21:17:59.440Z | onc-multi | DEBUG | Submitting request for page 9 (80 samples)...\n", - "2025-11-10T21:18:00.066Z | onc-service | INFO | Requested: https://data.oceannetworks.ca/api/scalardata?method=getByLocation&dateTo=2009-08-29T23%3A59%3A59.999Z&rowLimit=10&locationCode=BACVP&deviceCategoryCode=CTD&dateFrom=2009-08-27T01%3A00%3A32.910Z&token=REDACTED\n", - "2025-11-10T21:18:00.066Z | onc-service | DEBUG | Response received in 0.619 seconds.\n", - "2025-11-10T21:18:00.066Z | onc-service | INFO | HTTP Response: OK (200)\n", - "2025-11-10T21:18:00.074Z | onc-multi | DEBUG | Submitting request for page 10 (90 samples)...\n", - "2025-11-10T21:18:00.676Z | onc-service | INFO | Requested: https://data.oceannetworks.ca/api/scalardata?method=getByLocation&dateTo=2009-08-29T23%3A59%3A59.999Z&rowLimit=10&locationCode=BACVP&deviceCategoryCode=CTD&dateFrom=2009-08-27T01%3A01%3A32.909Z&token=REDACTED\n", - "2025-11-10T21:18:00.676Z | onc-service | DEBUG | Response received in 0.6 seconds.\n", - "2025-11-10T21:18:00.676Z | onc-service | INFO | HTTP Response: OK (200)\n", - "2025-11-10T21:18:00.687Z | onc-multi | INFO | Downloaded 94 total samples in 6 seconds.\n" - ] - }, - { - "data": { - "text/plain": [ - "[{'actualSamples': 3,\n", - " 'data': {'qaqcFlags': [0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0],\n", - " 'sampleTimes': ['2009-08-27T00:53:29.800Z',\n", - " '2009-08-27T00:53:30.043Z',\n", - " '2009-08-27T00:53:30.289Z',\n", - " '2009-08-27T00:53:30.539Z',\n", - " '2009-08-27T00:53:30.789Z',\n", - " '2009-08-27T00:53:31.039Z',\n", - " '2009-08-27T00:53:31.289Z',\n", - " '2009-08-27T00:53:31.539Z',\n", - " '2009-08-27T00:53:31.789Z',\n", - " '2009-08-27T00:53:32.039Z',\n", - " '2009-08-27T00:54:30.585Z',\n", - " '2009-08-27T00:54:30.834Z',\n", - " '2009-08-27T00:54:31.084Z',\n", - " '2009-08-27T00:54:31.333Z',\n", - " '2009-08-27T00:54:31.583Z',\n", - " '2009-08-27T00:54:31.833Z',\n", - " '2009-08-27T00:54:32.083Z',\n", - " '2009-08-27T00:54:32.333Z',\n", - " '2009-08-27T00:54:32.583Z',\n", - " '2009-08-27T00:54:32.833Z',\n", - " '2009-08-27T00:55:30.578Z',\n", - " '2009-08-27T00:55:30.827Z',\n", - " '2009-08-27T00:55:31.077Z',\n", - " '2009-08-27T00:55:31.325Z',\n", - " '2009-08-27T00:55:31.575Z',\n", - " '2009-08-27T00:55:31.825Z',\n", - " '2009-08-27T00:55:32.075Z',\n", - " '2009-08-27T00:55:32.325Z',\n", - " '2009-08-27T00:55:32.575Z',\n", - " '2009-08-27T00:55:32.825Z',\n", - " '2009-08-27T00:56:30.597Z',\n", - " '2009-08-27T00:56:30.847Z',\n", - " '2009-08-27T00:56:31.033Z',\n", - " '2009-08-27T00:56:31.225Z',\n", - " '2009-08-27T00:56:31.475Z',\n", - " '2009-08-27T00:56:31.725Z',\n", - " '2009-08-27T00:56:31.975Z',\n", - " '2009-08-27T00:56:32.225Z',\n", - " '2009-08-27T00:56:32.475Z',\n", - " '2009-08-27T00:56:32.725Z',\n", - " '2009-08-27T00:57:30.380Z',\n", - " '2009-08-27T00:57:30.631Z',\n", - " '2009-08-27T00:57:30.882Z',\n", - " '2009-08-27T00:57:31.131Z',\n", - " '2009-08-27T00:57:31.381Z',\n", - " '2009-08-27T00:57:31.631Z',\n", - " '2009-08-27T00:57:31.881Z',\n", - " '2009-08-27T00:57:32.131Z',\n", - " '2009-08-27T00:57:32.381Z',\n", - " '2009-08-27T00:57:32.631Z',\n", - " '2009-08-27T00:58:30.387Z',\n", - " '2009-08-27T00:58:30.638Z',\n", - " '2009-08-27T00:58:30.889Z',\n", - " '2009-08-27T00:58:31.141Z',\n", - " '2009-08-27T00:58:31.391Z',\n", - " '2009-08-27T00:58:31.641Z',\n", - " '2009-08-27T00:58:31.891Z',\n", - " '2009-08-27T00:58:32.141Z',\n", - " '2009-08-27T00:58:32.391Z',\n", - " '2009-08-27T00:58:32.641Z',\n", - " '2009-08-27T00:59:30.389Z',\n", - " '2009-08-27T00:59:30.639Z',\n", - " '2009-08-27T00:59:30.888Z',\n", - " '2009-08-27T00:59:31.136Z',\n", - " '2009-08-27T00:59:31.386Z',\n", - " '2009-08-27T00:59:31.636Z',\n", - " '2009-08-27T00:59:31.886Z',\n", - " '2009-08-27T00:59:32.136Z',\n", - " '2009-08-27T00:59:32.386Z',\n", - " '2009-08-27T00:59:32.636Z',\n", - " '2009-08-27T01:00:30.407Z',\n", - " '2009-08-27T01:00:30.657Z',\n", - " '2009-08-27T01:00:30.907Z',\n", - " '2009-08-27T01:00:31.160Z',\n", - " '2009-08-27T01:00:31.410Z',\n", - " '2009-08-27T01:00:31.660Z',\n", - " '2009-08-27T01:00:31.910Z',\n", - " '2009-08-27T01:00:32.160Z',\n", - " '2009-08-27T01:00:32.410Z',\n", - " '2009-08-27T01:00:32.660Z',\n", - " '2009-08-27T01:01:30.408Z',\n", - " '2009-08-27T01:01:30.657Z',\n", - " '2009-08-27T01:01:30.909Z',\n", - " '2009-08-27T01:01:31.159Z',\n", - " '2009-08-27T01:01:31.409Z',\n", - " '2009-08-27T01:01:31.659Z',\n", - " '2009-08-27T01:01:31.909Z',\n", - " '2009-08-27T01:01:32.159Z',\n", - " '2009-08-27T01:01:32.409Z',\n", - " '2009-08-27T01:01:32.659Z',\n", - " '2009-08-27T01:02:30.424Z',\n", - " '2009-08-27T01:02:30.674Z',\n", - " '2009-08-27T01:02:30.923Z',\n", - " '2009-08-27T01:02:31.173Z'],\n", - " 'values': [3.35513,\n", - " 3.35514,\n", - " 3.35505,\n", - " 3.35503,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 3.35511,\n", - " 3.35509,\n", - " 3.35506,\n", - " 3.35504,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 3.35504,\n", - " 3.35507,\n", - " 3.35511,\n", - " 3.35513,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 3.35501,\n", - " 3.35496,\n", - " 3.35495,\n", - " 3.35499,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 3.35499,\n", - " 3.35498,\n", - " 3.35496,\n", - " 3.35485,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 3.35488,\n", - " 3.35482,\n", - " 3.35477,\n", - " 3.35476,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 3.35476,\n", - " 3.35473,\n", - " 3.35464,\n", - " 3.3546,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 3.35469,\n", - " 3.35464,\n", - " 3.35463,\n", - " 3.35456,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 3.35466,\n", - " 3.35465,\n", - " 3.35462,\n", - " 3.35463,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 3.35462,\n", - " 3.35461,\n", - " 3.3546,\n", - " 3.35458]},\n", - " 'outputFormat': 'array',\n", - " 'propertyCode': 'conductivity',\n", - " 'sensorCategoryCode': 'conductivity',\n", - " 'sensorCode': 'cond',\n", - " 'sensorName': 'Conductivity',\n", - " 'unitOfMeasure': 'S/m'},\n", - " {'actualSamples': 3,\n", - " 'data': {'qaqcFlags': [0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0],\n", - " 'sampleTimes': ['2009-08-27T00:53:29.800Z',\n", - " '2009-08-27T00:53:30.043Z',\n", - " '2009-08-27T00:53:30.289Z',\n", - " '2009-08-27T00:53:30.539Z',\n", - " '2009-08-27T00:53:30.789Z',\n", - " '2009-08-27T00:53:31.039Z',\n", - " '2009-08-27T00:53:31.289Z',\n", - " '2009-08-27T00:53:31.539Z',\n", - " '2009-08-27T00:53:31.789Z',\n", - " '2009-08-27T00:53:32.039Z',\n", - " '2009-08-27T00:54:30.585Z',\n", - " '2009-08-27T00:54:30.834Z',\n", - " '2009-08-27T00:54:31.084Z',\n", - " '2009-08-27T00:54:31.333Z',\n", - " '2009-08-27T00:54:31.583Z',\n", - " '2009-08-27T00:54:31.833Z',\n", - " '2009-08-27T00:54:32.083Z',\n", - " '2009-08-27T00:54:32.333Z',\n", - " '2009-08-27T00:54:32.583Z',\n", - " '2009-08-27T00:54:32.833Z',\n", - " '2009-08-27T00:55:30.578Z',\n", - " '2009-08-27T00:55:30.827Z',\n", - " '2009-08-27T00:55:31.077Z',\n", - " '2009-08-27T00:55:31.325Z',\n", - " '2009-08-27T00:55:31.575Z',\n", - " '2009-08-27T00:55:31.825Z',\n", - " '2009-08-27T00:55:32.075Z',\n", - " '2009-08-27T00:55:32.325Z',\n", - " '2009-08-27T00:55:32.575Z',\n", - " '2009-08-27T00:55:32.825Z',\n", - " '2009-08-27T00:56:30.597Z',\n", - " '2009-08-27T00:56:30.847Z',\n", - " '2009-08-27T00:56:31.033Z',\n", - " '2009-08-27T00:56:31.225Z',\n", - " '2009-08-27T00:56:31.475Z',\n", - " '2009-08-27T00:56:31.725Z',\n", - " '2009-08-27T00:56:31.975Z',\n", - " '2009-08-27T00:56:32.225Z',\n", - " '2009-08-27T00:56:32.475Z',\n", - " '2009-08-27T00:56:32.725Z',\n", - " '2009-08-27T00:57:30.380Z',\n", - " '2009-08-27T00:57:30.631Z',\n", - " '2009-08-27T00:57:30.882Z',\n", - " '2009-08-27T00:57:31.131Z',\n", - " '2009-08-27T00:57:31.381Z',\n", - " '2009-08-27T00:57:31.631Z',\n", - " '2009-08-27T00:57:31.881Z',\n", - " '2009-08-27T00:57:32.131Z',\n", - " '2009-08-27T00:57:32.381Z',\n", - " '2009-08-27T00:57:32.631Z',\n", - " '2009-08-27T00:58:30.387Z',\n", - " '2009-08-27T00:58:30.638Z',\n", - " '2009-08-27T00:58:30.889Z',\n", - " '2009-08-27T00:58:31.141Z',\n", - " '2009-08-27T00:58:31.391Z',\n", - " '2009-08-27T00:58:31.641Z',\n", - " '2009-08-27T00:58:31.891Z',\n", - " '2009-08-27T00:58:32.141Z',\n", - " '2009-08-27T00:58:32.391Z',\n", - " '2009-08-27T00:58:32.641Z',\n", - " '2009-08-27T00:59:30.389Z',\n", - " '2009-08-27T00:59:30.639Z',\n", - " '2009-08-27T00:59:30.888Z',\n", - " '2009-08-27T00:59:31.136Z',\n", - " '2009-08-27T00:59:31.386Z',\n", - " '2009-08-27T00:59:31.636Z',\n", - " '2009-08-27T00:59:31.886Z',\n", - " '2009-08-27T00:59:32.136Z',\n", - " '2009-08-27T00:59:32.386Z',\n", - " '2009-08-27T00:59:32.636Z',\n", - " '2009-08-27T01:00:30.407Z',\n", - " '2009-08-27T01:00:30.657Z',\n", - " '2009-08-27T01:00:30.907Z',\n", - " '2009-08-27T01:00:31.160Z',\n", - " '2009-08-27T01:00:31.410Z',\n", - " '2009-08-27T01:00:31.660Z',\n", - " '2009-08-27T01:00:31.910Z',\n", - " '2009-08-27T01:00:32.160Z',\n", - " '2009-08-27T01:00:32.410Z',\n", - " '2009-08-27T01:00:32.660Z',\n", - " '2009-08-27T01:01:30.408Z',\n", - " '2009-08-27T01:01:30.657Z',\n", - " '2009-08-27T01:01:30.909Z',\n", - " '2009-08-27T01:01:31.159Z',\n", - " '2009-08-27T01:01:31.409Z',\n", - " '2009-08-27T01:01:31.659Z',\n", - " '2009-08-27T01:01:31.909Z',\n", - " '2009-08-27T01:01:32.159Z',\n", - " '2009-08-27T01:01:32.409Z',\n", - " '2009-08-27T01:01:32.659Z',\n", - " '2009-08-27T01:02:30.424Z',\n", - " '2009-08-27T01:02:30.674Z',\n", - " '2009-08-27T01:02:30.923Z',\n", - " '2009-08-27T01:02:31.173Z'],\n", - " 'values': [393.13,\n", - " 397.98,\n", - " 388.16,\n", - " 387.96,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 457.53,\n", - " 426.99,\n", - " 422.17,\n", - " 403.53,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 342.4,\n", - " 384.93,\n", - " 633.38,\n", - " 701.15,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 305.69,\n", - " 186.96,\n", - " 205.94,\n", - " 268.58,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 326.42,\n", - " 247.48,\n", - " 231.35,\n", - " 214.03,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 316.57,\n", - " 265.31,\n", - " 258.51,\n", - " 255.77,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 317.38,\n", - " 272.14,\n", - " 269.27,\n", - " 266.25,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 338.83,\n", - " 277.15,\n", - " 273.76,\n", - " 271.97,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 339.87,\n", - " 279.94,\n", - " 277.44,\n", - " 274.57,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 340.92,\n", - " 281.16,\n", - " 279.27,\n", - " 277.16]},\n", - " 'outputFormat': 'array',\n", - " 'propertyCode': 'depth',\n", - " 'sensorCategoryCode': 'depth',\n", - " 'sensorCode': 'depth',\n", - " 'sensorName': 'Depth',\n", - " 'unitOfMeasure': 'm'},\n", - " {'actualSamples': 3,\n", - " 'data': {'qaqcFlags': [0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0],\n", - " 'sampleTimes': ['2009-08-27T00:53:29.800Z',\n", - " '2009-08-27T00:53:30.043Z',\n", - " '2009-08-27T00:53:30.289Z',\n", - " '2009-08-27T00:53:30.539Z',\n", - " '2009-08-27T00:53:30.789Z',\n", - " '2009-08-27T00:53:31.039Z',\n", - " '2009-08-27T00:53:31.289Z',\n", - " '2009-08-27T00:53:31.539Z',\n", - " '2009-08-27T00:53:31.789Z',\n", - " '2009-08-27T00:53:32.039Z',\n", - " '2009-08-27T00:54:30.585Z',\n", - " '2009-08-27T00:54:30.834Z',\n", - " '2009-08-27T00:54:31.084Z',\n", - " '2009-08-27T00:54:31.333Z',\n", - " '2009-08-27T00:54:31.583Z',\n", - " '2009-08-27T00:54:31.833Z',\n", - " '2009-08-27T00:54:32.083Z',\n", - " '2009-08-27T00:54:32.333Z',\n", - " '2009-08-27T00:54:32.583Z',\n", - " '2009-08-27T00:54:32.833Z',\n", - " '2009-08-27T00:55:30.578Z',\n", - " '2009-08-27T00:55:30.827Z',\n", - " '2009-08-27T00:55:31.077Z',\n", - " '2009-08-27T00:55:31.325Z',\n", - " '2009-08-27T00:55:31.575Z',\n", - " '2009-08-27T00:55:31.825Z',\n", - " '2009-08-27T00:55:32.075Z',\n", - " '2009-08-27T00:55:32.325Z',\n", - " '2009-08-27T00:55:32.575Z',\n", - " '2009-08-27T00:55:32.825Z',\n", - " '2009-08-27T00:56:30.597Z',\n", - " '2009-08-27T00:56:30.847Z',\n", - " '2009-08-27T00:56:31.033Z',\n", - " '2009-08-27T00:56:31.225Z',\n", - " '2009-08-27T00:56:31.475Z',\n", - " '2009-08-27T00:56:31.725Z',\n", - " '2009-08-27T00:56:31.975Z',\n", - " '2009-08-27T00:56:32.225Z',\n", - " '2009-08-27T00:56:32.475Z',\n", - " '2009-08-27T00:56:32.725Z',\n", - " '2009-08-27T00:57:30.380Z',\n", - " '2009-08-27T00:57:30.631Z',\n", - " '2009-08-27T00:57:30.882Z',\n", - " '2009-08-27T00:57:31.131Z',\n", - " '2009-08-27T00:57:31.381Z',\n", - " '2009-08-27T00:57:31.631Z',\n", - " '2009-08-27T00:57:31.881Z',\n", - " '2009-08-27T00:57:32.131Z',\n", - " '2009-08-27T00:57:32.381Z',\n", - " '2009-08-27T00:57:32.631Z',\n", - " '2009-08-27T00:58:30.387Z',\n", - " '2009-08-27T00:58:30.638Z',\n", - " '2009-08-27T00:58:30.889Z',\n", - " '2009-08-27T00:58:31.141Z',\n", - " '2009-08-27T00:58:31.391Z',\n", - " '2009-08-27T00:58:31.641Z',\n", - " '2009-08-27T00:58:31.891Z',\n", - " '2009-08-27T00:58:32.141Z',\n", - " '2009-08-27T00:58:32.391Z',\n", - " '2009-08-27T00:58:32.641Z',\n", - " '2009-08-27T00:59:30.389Z',\n", - " '2009-08-27T00:59:30.639Z',\n", - " '2009-08-27T00:59:30.888Z',\n", - " '2009-08-27T00:59:31.136Z',\n", - " '2009-08-27T00:59:31.386Z',\n", - " '2009-08-27T00:59:31.636Z',\n", - " '2009-08-27T00:59:31.886Z',\n", - " '2009-08-27T00:59:32.136Z',\n", - " '2009-08-27T00:59:32.386Z',\n", - " '2009-08-27T00:59:32.636Z',\n", - " '2009-08-27T01:00:30.407Z',\n", - " '2009-08-27T01:00:30.657Z',\n", - " '2009-08-27T01:00:30.907Z',\n", - " '2009-08-27T01:00:31.160Z',\n", - " '2009-08-27T01:00:31.410Z',\n", - " '2009-08-27T01:00:31.660Z',\n", - " '2009-08-27T01:00:31.910Z',\n", - " '2009-08-27T01:00:32.160Z',\n", - " '2009-08-27T01:00:32.410Z',\n", - " '2009-08-27T01:00:32.660Z',\n", - " '2009-08-27T01:01:30.408Z',\n", - " '2009-08-27T01:01:30.657Z',\n", - " '2009-08-27T01:01:30.909Z',\n", - " '2009-08-27T01:01:31.159Z',\n", - " '2009-08-27T01:01:31.409Z',\n", - " '2009-08-27T01:01:31.659Z',\n", - " '2009-08-27T01:01:31.909Z',\n", - " '2009-08-27T01:01:32.159Z',\n", - " '2009-08-27T01:01:32.409Z',\n", - " '2009-08-27T01:01:32.659Z',\n", - " '2009-08-27T01:02:30.424Z',\n", - " '2009-08-27T01:02:30.674Z',\n", - " '2009-08-27T01:02:30.923Z',\n", - " '2009-08-27T01:02:31.173Z'],\n", - " 'values': [396.856,\n", - " 401.752,\n", - " 391.834,\n", - " 391.625,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 461.933,\n", - " 431.072,\n", - " 426.201,\n", - " 407.362,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 345.595,\n", - " 388.571,\n", - " 639.748,\n", - " 708.315,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 308.518,\n", - " 188.634,\n", - " 207.793,\n", - " 271.046,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 329.454,\n", - " 249.735,\n", - " 233.447,\n", - " 215.966,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 319.506,\n", - " 267.735,\n", - " 260.868,\n", - " 258.11,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 320.33,\n", - " 274.635,\n", - " 271.74,\n", - " 268.684,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 341.994,\n", - " 279.7,\n", - " 276.276,\n", - " 274.464,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 343.043,\n", - " 282.517,\n", - " 279.985,\n", - " 277.087,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 344.1,\n", - " 283.742,\n", - " 281.84,\n", - " 279.701]},\n", - " 'outputFormat': 'array',\n", - " 'propertyCode': 'pressure',\n", - " 'sensorCategoryCode': 'pressure',\n", - " 'sensorCode': 'Pressure',\n", - " 'sensorName': 'Pressure',\n", - " 'unitOfMeasure': 'decibar'},\n", - " {'actualSamples': 3,\n", - " 'data': {'qaqcFlags': [0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0],\n", - " 'sampleTimes': ['2009-08-27T00:53:29.800Z',\n", - " '2009-08-27T00:53:30.043Z',\n", - " '2009-08-27T00:53:30.289Z',\n", - " '2009-08-27T00:53:30.539Z',\n", - " '2009-08-27T00:53:30.789Z',\n", - " '2009-08-27T00:53:31.039Z',\n", - " '2009-08-27T00:53:31.289Z',\n", - " '2009-08-27T00:53:31.539Z',\n", - " '2009-08-27T00:53:31.789Z',\n", - " '2009-08-27T00:53:32.039Z',\n", - " '2009-08-27T00:54:30.585Z',\n", - " '2009-08-27T00:54:30.834Z',\n", - " '2009-08-27T00:54:31.084Z',\n", - " '2009-08-27T00:54:31.333Z',\n", - " '2009-08-27T00:54:31.583Z',\n", - " '2009-08-27T00:54:31.833Z',\n", - " '2009-08-27T00:54:32.083Z',\n", - " '2009-08-27T00:54:32.333Z',\n", - " '2009-08-27T00:54:32.583Z',\n", - " '2009-08-27T00:54:32.833Z',\n", - " '2009-08-27T00:55:30.578Z',\n", - " '2009-08-27T00:55:30.827Z',\n", - " '2009-08-27T00:55:31.077Z',\n", - " '2009-08-27T00:55:31.325Z',\n", - " '2009-08-27T00:55:31.575Z',\n", - " '2009-08-27T00:55:31.825Z',\n", - " '2009-08-27T00:55:32.075Z',\n", - " '2009-08-27T00:55:32.325Z',\n", - " '2009-08-27T00:55:32.575Z',\n", - " '2009-08-27T00:55:32.825Z',\n", - " '2009-08-27T00:56:30.597Z',\n", - " '2009-08-27T00:56:30.847Z',\n", - " '2009-08-27T00:56:31.033Z',\n", - " '2009-08-27T00:56:31.225Z',\n", - " '2009-08-27T00:56:31.475Z',\n", - " '2009-08-27T00:56:31.725Z',\n", - " '2009-08-27T00:56:31.975Z',\n", - " '2009-08-27T00:56:32.225Z',\n", - " '2009-08-27T00:56:32.475Z',\n", - " '2009-08-27T00:56:32.725Z',\n", - " '2009-08-27T00:57:30.380Z',\n", - " '2009-08-27T00:57:30.631Z',\n", - " '2009-08-27T00:57:30.882Z',\n", - " '2009-08-27T00:57:31.131Z',\n", - " '2009-08-27T00:57:31.381Z',\n", - " '2009-08-27T00:57:31.631Z',\n", - " '2009-08-27T00:57:31.881Z',\n", - " '2009-08-27T00:57:32.131Z',\n", - " '2009-08-27T00:57:32.381Z',\n", - " '2009-08-27T00:57:32.631Z',\n", - " '2009-08-27T00:58:30.387Z',\n", - " '2009-08-27T00:58:30.638Z',\n", - " '2009-08-27T00:58:30.889Z',\n", - " '2009-08-27T00:58:31.141Z',\n", - " '2009-08-27T00:58:31.391Z',\n", - " '2009-08-27T00:58:31.641Z',\n", - " '2009-08-27T00:58:31.891Z',\n", - " '2009-08-27T00:58:32.141Z',\n", - " '2009-08-27T00:58:32.391Z',\n", - " '2009-08-27T00:58:32.641Z',\n", - " '2009-08-27T00:59:30.389Z',\n", - " '2009-08-27T00:59:30.639Z',\n", - " '2009-08-27T00:59:30.888Z',\n", - " '2009-08-27T00:59:31.136Z',\n", - " '2009-08-27T00:59:31.386Z',\n", - " '2009-08-27T00:59:31.636Z',\n", - " '2009-08-27T00:59:31.886Z',\n", - " '2009-08-27T00:59:32.136Z',\n", - " '2009-08-27T00:59:32.386Z',\n", - " '2009-08-27T00:59:32.636Z',\n", - " '2009-08-27T01:00:30.407Z',\n", - " '2009-08-27T01:00:30.657Z',\n", - " '2009-08-27T01:00:30.907Z',\n", - " '2009-08-27T01:00:31.160Z',\n", - " '2009-08-27T01:00:31.410Z',\n", - " '2009-08-27T01:00:31.660Z',\n", - " '2009-08-27T01:00:31.910Z',\n", - " '2009-08-27T01:00:32.160Z',\n", - " '2009-08-27T01:00:32.410Z',\n", - " '2009-08-27T01:00:32.660Z',\n", - " '2009-08-27T01:01:30.408Z',\n", - " '2009-08-27T01:01:30.657Z',\n", - " '2009-08-27T01:01:30.909Z',\n", - " '2009-08-27T01:01:31.159Z',\n", - " '2009-08-27T01:01:31.409Z',\n", - " '2009-08-27T01:01:31.659Z',\n", - " '2009-08-27T01:01:31.909Z',\n", - " '2009-08-27T01:01:32.159Z',\n", - " '2009-08-27T01:01:32.409Z',\n", - " '2009-08-27T01:01:32.659Z',\n", - " '2009-08-27T01:02:30.424Z',\n", - " '2009-08-27T01:02:30.674Z',\n", - " '2009-08-27T01:02:30.923Z',\n", - " '2009-08-27T01:02:31.173Z'],\n", - " 'values': [34.0584,\n", - " 34.0562,\n", - " 34.0601,\n", - " 34.06,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 34.0262,\n", - " 34.0414,\n", - " 34.0432,\n", - " 34.0523,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 34.0837,\n", - " 34.0627,\n", - " 33.9396,\n", - " 33.9069,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 34.1021,\n", - " 34.1618,\n", - " 34.152,\n", - " 34.1208,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 34.0914,\n", - " 34.1315,\n", - " 34.1392,\n", - " 34.1472,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 34.0973,\n", - " 34.1225,\n", - " 34.1254,\n", - " 34.1267,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 34.0975,\n", - " 34.12,\n", - " 34.1205,\n", - " 34.1217,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 34.0859,\n", - " 34.1166,\n", - " 34.1181,\n", - " 34.1183,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 34.085,\n", - " 34.1149,\n", - " 34.1161,\n", - " 34.1175,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 34.0847,\n", - " 34.1148,\n", - " 34.1156,\n", - " 34.1165]},\n", - " 'outputFormat': 'array',\n", - " 'propertyCode': 'salinity',\n", - " 'sensorCategoryCode': 'salinity',\n", - " 'sensorCode': 'salinity',\n", - " 'sensorName': 'Practical Salinity',\n", - " 'unitOfMeasure': 'psu'},\n", - " {'actualSamples': 3,\n", - " 'data': {'qaqcFlags': [0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0],\n", - " 'sampleTimes': ['2009-08-27T00:53:29.800Z',\n", - " '2009-08-27T00:53:30.043Z',\n", - " '2009-08-27T00:53:30.289Z',\n", - " '2009-08-27T00:53:30.539Z',\n", - " '2009-08-27T00:53:30.789Z',\n", - " '2009-08-27T00:53:31.039Z',\n", - " '2009-08-27T00:53:31.289Z',\n", - " '2009-08-27T00:53:31.539Z',\n", - " '2009-08-27T00:53:31.789Z',\n", - " '2009-08-27T00:53:32.039Z',\n", - " '2009-08-27T00:54:30.585Z',\n", - " '2009-08-27T00:54:30.834Z',\n", - " '2009-08-27T00:54:31.084Z',\n", - " '2009-08-27T00:54:31.333Z',\n", - " '2009-08-27T00:54:31.583Z',\n", - " '2009-08-27T00:54:31.833Z',\n", - " '2009-08-27T00:54:32.083Z',\n", - " '2009-08-27T00:54:32.333Z',\n", - " '2009-08-27T00:54:32.583Z',\n", - " '2009-08-27T00:54:32.833Z',\n", - " '2009-08-27T00:55:30.578Z',\n", - " '2009-08-27T00:55:30.827Z',\n", - " '2009-08-27T00:55:31.077Z',\n", - " '2009-08-27T00:55:31.325Z',\n", - " '2009-08-27T00:55:31.575Z',\n", - " '2009-08-27T00:55:31.825Z',\n", - " '2009-08-27T00:55:32.075Z',\n", - " '2009-08-27T00:55:32.325Z',\n", - " '2009-08-27T00:55:32.575Z',\n", - " '2009-08-27T00:55:32.825Z',\n", - " '2009-08-27T00:56:30.597Z',\n", - " '2009-08-27T00:56:30.847Z',\n", - " '2009-08-27T00:56:31.033Z',\n", - " '2009-08-27T00:56:31.225Z',\n", - " '2009-08-27T00:56:31.475Z',\n", - " '2009-08-27T00:56:31.725Z',\n", - " '2009-08-27T00:56:31.975Z',\n", - " '2009-08-27T00:56:32.225Z',\n", - " '2009-08-27T00:56:32.475Z',\n", - " '2009-08-27T00:56:32.725Z',\n", - " '2009-08-27T00:57:30.380Z',\n", - " '2009-08-27T00:57:30.631Z',\n", - " '2009-08-27T00:57:30.882Z',\n", - " '2009-08-27T00:57:31.131Z',\n", - " '2009-08-27T00:57:31.381Z',\n", - " '2009-08-27T00:57:31.631Z',\n", - " '2009-08-27T00:57:31.881Z',\n", - " '2009-08-27T00:57:32.131Z',\n", - " '2009-08-27T00:57:32.381Z',\n", - " '2009-08-27T00:57:32.631Z',\n", - " '2009-08-27T00:58:30.387Z',\n", - " '2009-08-27T00:58:30.638Z',\n", - " '2009-08-27T00:58:30.889Z',\n", - " '2009-08-27T00:58:31.141Z',\n", - " '2009-08-27T00:58:31.391Z',\n", - " '2009-08-27T00:58:31.641Z',\n", - " '2009-08-27T00:58:31.891Z',\n", - " '2009-08-27T00:58:32.141Z',\n", - " '2009-08-27T00:58:32.391Z',\n", - " '2009-08-27T00:58:32.641Z',\n", - " '2009-08-27T00:59:30.389Z',\n", - " '2009-08-27T00:59:30.639Z',\n", - " '2009-08-27T00:59:30.888Z',\n", - " '2009-08-27T00:59:31.136Z',\n", - " '2009-08-27T00:59:31.386Z',\n", - " '2009-08-27T00:59:31.636Z',\n", - " '2009-08-27T00:59:31.886Z',\n", - " '2009-08-27T00:59:32.136Z',\n", - " '2009-08-27T00:59:32.386Z',\n", - " '2009-08-27T00:59:32.636Z',\n", - " '2009-08-27T01:00:30.407Z',\n", - " '2009-08-27T01:00:30.657Z',\n", - " '2009-08-27T01:00:30.907Z',\n", - " '2009-08-27T01:00:31.160Z',\n", - " '2009-08-27T01:00:31.410Z',\n", - " '2009-08-27T01:00:31.660Z',\n", - " '2009-08-27T01:00:31.910Z',\n", - " '2009-08-27T01:00:32.160Z',\n", - " '2009-08-27T01:00:32.410Z',\n", - " '2009-08-27T01:00:32.660Z',\n", - " '2009-08-27T01:01:30.408Z',\n", - " '2009-08-27T01:01:30.657Z',\n", - " '2009-08-27T01:01:30.909Z',\n", - " '2009-08-27T01:01:31.159Z',\n", - " '2009-08-27T01:01:31.409Z',\n", - " '2009-08-27T01:01:31.659Z',\n", - " '2009-08-27T01:01:31.909Z',\n", - " '2009-08-27T01:01:32.159Z',\n", - " '2009-08-27T01:01:32.409Z',\n", - " '2009-08-27T01:01:32.659Z',\n", - " '2009-08-27T01:02:30.424Z',\n", - " '2009-08-27T01:02:30.674Z',\n", - " '2009-08-27T01:02:30.923Z',\n", - " '2009-08-27T01:02:31.173Z'],\n", - " 'values': [26.83482199383866,\n", - " 26.833135110028707,\n", - " 26.836124468444496,\n", - " 26.83604313520459,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 26.810075942027424,\n", - " 26.821797794790882,\n", - " 26.82314384078154,\n", - " 26.830136239442254,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 26.854376133816686,\n", - " 26.838231537027696,\n", - " 26.743609425131126,\n", - " 26.7185395999486,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 26.868567078853857,\n", - " 26.914495190317666,\n", - " 26.906947076811548,\n", - " 26.88296826678652,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 26.860317002403917,\n", - " 26.89120368114618,\n", - " 26.897095706605114,\n", - " 26.90329931409292,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 26.865147630873935,\n", - " 26.884512111930007,\n", - " 26.886732587094002,\n", - " 26.887743736117272,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 26.86554888761725,\n", - " 26.88285528766187,\n", - " 26.88323232524158,\n", - " 26.884161211195078,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 26.856607287764973,\n", - " 26.88023273309932,\n", - " 26.881382567675928,\n", - " 26.88153385031319,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 26.855906804638153,\n", - " 26.878868945131217,\n", - " 26.879828027172834,\n", - " 26.8808920177878,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 26.85576714051558,\n", - " 26.878938635334407,\n", - " 26.879538725086377,\n", - " 26.880227723996086]},\n", - " 'outputFormat': 'array',\n", - " 'propertyCode': 'sigmatheta',\n", - " 'sensorCategoryCode': 'sigma_theta',\n", - " 'sensorCode': 'SIGMA_THETA',\n", - " 'sensorName': 'Sigma-theta (0 dbar)',\n", - " 'unitOfMeasure': 'kg/m3'},\n", - " {'actualSamples': 3,\n", - " 'data': {'qaqcFlags': [0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 9,\n", - " 0,\n", - " 0,\n", - " 0,\n", - " 0],\n", - " 'sampleTimes': ['2009-08-27T00:53:29.800Z',\n", - " '2009-08-27T00:53:30.043Z',\n", - " '2009-08-27T00:53:30.289Z',\n", - " '2009-08-27T00:53:30.539Z',\n", - " '2009-08-27T00:53:30.789Z',\n", - " '2009-08-27T00:53:31.039Z',\n", - " '2009-08-27T00:53:31.289Z',\n", - " '2009-08-27T00:53:31.539Z',\n", - " '2009-08-27T00:53:31.789Z',\n", - " '2009-08-27T00:53:32.039Z',\n", - " '2009-08-27T00:54:30.585Z',\n", - " '2009-08-27T00:54:30.834Z',\n", - " '2009-08-27T00:54:31.084Z',\n", - " '2009-08-27T00:54:31.333Z',\n", - " '2009-08-27T00:54:31.583Z',\n", - " '2009-08-27T00:54:31.833Z',\n", - " '2009-08-27T00:54:32.083Z',\n", - " '2009-08-27T00:54:32.333Z',\n", - " '2009-08-27T00:54:32.583Z',\n", - " '2009-08-27T00:54:32.833Z',\n", - " '2009-08-27T00:55:30.578Z',\n", - " '2009-08-27T00:55:30.827Z',\n", - " '2009-08-27T00:55:31.077Z',\n", - " '2009-08-27T00:55:31.325Z',\n", - " '2009-08-27T00:55:31.575Z',\n", - " '2009-08-27T00:55:31.825Z',\n", - " '2009-08-27T00:55:32.075Z',\n", - " '2009-08-27T00:55:32.325Z',\n", - " '2009-08-27T00:55:32.575Z',\n", - " '2009-08-27T00:55:32.825Z',\n", - " '2009-08-27T00:56:30.597Z',\n", - " '2009-08-27T00:56:30.847Z',\n", - " '2009-08-27T00:56:31.033Z',\n", - " '2009-08-27T00:56:31.225Z',\n", - " '2009-08-27T00:56:31.475Z',\n", - " '2009-08-27T00:56:31.725Z',\n", - " '2009-08-27T00:56:31.975Z',\n", - " '2009-08-27T00:56:32.225Z',\n", - " '2009-08-27T00:56:32.475Z',\n", - " '2009-08-27T00:56:32.725Z',\n", - " '2009-08-27T00:57:30.380Z',\n", - " '2009-08-27T00:57:30.631Z',\n", - " '2009-08-27T00:57:30.882Z',\n", - " '2009-08-27T00:57:31.131Z',\n", - " '2009-08-27T00:57:31.381Z',\n", - " '2009-08-27T00:57:31.631Z',\n", - " '2009-08-27T00:57:31.881Z',\n", - " '2009-08-27T00:57:32.131Z',\n", - " '2009-08-27T00:57:32.381Z',\n", - " '2009-08-27T00:57:32.631Z',\n", - " '2009-08-27T00:58:30.387Z',\n", - " '2009-08-27T00:58:30.638Z',\n", - " '2009-08-27T00:58:30.889Z',\n", - " '2009-08-27T00:58:31.141Z',\n", - " '2009-08-27T00:58:31.391Z',\n", - " '2009-08-27T00:58:31.641Z',\n", - " '2009-08-27T00:58:31.891Z',\n", - " '2009-08-27T00:58:32.141Z',\n", - " '2009-08-27T00:58:32.391Z',\n", - " '2009-08-27T00:58:32.641Z',\n", - " '2009-08-27T00:59:30.389Z',\n", - " '2009-08-27T00:59:30.639Z',\n", - " '2009-08-27T00:59:30.888Z',\n", - " '2009-08-27T00:59:31.136Z',\n", - " '2009-08-27T00:59:31.386Z',\n", - " '2009-08-27T00:59:31.636Z',\n", - " '2009-08-27T00:59:31.886Z',\n", - " '2009-08-27T00:59:32.136Z',\n", - " '2009-08-27T00:59:32.386Z',\n", - " '2009-08-27T00:59:32.636Z',\n", - " '2009-08-27T01:00:30.407Z',\n", - " '2009-08-27T01:00:30.657Z',\n", - " '2009-08-27T01:00:30.907Z',\n", - " '2009-08-27T01:00:31.160Z',\n", - " '2009-08-27T01:00:31.410Z',\n", - " '2009-08-27T01:00:31.660Z',\n", - " '2009-08-27T01:00:31.910Z',\n", - " '2009-08-27T01:00:32.160Z',\n", - " '2009-08-27T01:00:32.410Z',\n", - " '2009-08-27T01:00:32.660Z',\n", - " '2009-08-27T01:01:30.408Z',\n", - " '2009-08-27T01:01:30.657Z',\n", - " '2009-08-27T01:01:30.909Z',\n", - " '2009-08-27T01:01:31.159Z',\n", - " '2009-08-27T01:01:31.409Z',\n", - " '2009-08-27T01:01:31.659Z',\n", - " '2009-08-27T01:01:31.909Z',\n", - " '2009-08-27T01:01:32.159Z',\n", - " '2009-08-27T01:01:32.409Z',\n", - " '2009-08-27T01:01:32.659Z',\n", - " '2009-08-27T01:02:30.424Z',\n", - " '2009-08-27T01:02:30.674Z',\n", - " '2009-08-27T01:02:30.923Z',\n", - " '2009-08-27T01:02:31.173Z'],\n", - " 'values': [5.8232,\n", - " 5.8232,\n", - " 5.8231,\n", - " 5.8231,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 5.8231,\n", - " 5.8228,\n", - " 5.823,\n", - " 5.823,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 5.8224,\n", - " 5.8224,\n", - " 5.8226,\n", - " 5.8224,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 5.8221,\n", - " 5.8222,\n", - " 5.8222,\n", - " 5.822,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 5.8222,\n", - " 5.822,\n", - " 5.8222,\n", - " 5.8217,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 5.82,\n", - " 5.8201,\n", - " 5.8201,\n", - " 5.82,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 5.8181,\n", - " 5.8181,\n", - " 5.818,\n", - " 5.8179,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 5.8181,\n", - " 5.818,\n", - " 5.818,\n", - " 5.8179,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 5.8181,\n", - " 5.8184,\n", - " 5.8181,\n", - " 5.8182,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " nan,\n", - " 5.8174,\n", - " 5.8173,\n", - " 5.8174,\n", - " 5.8174]},\n", - " 'outputFormat': 'array',\n", - " 'propertyCode': 'seawatertemperature',\n", - " 'sensorCategoryCode': 'temperature',\n", - " 'sensorCode': 'Temperature',\n", - " 'sensorName': 'Temperature',\n", - " 'unitOfMeasure': 'C'}]" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "execution_count": 7 - }, - { - "metadata": { - "ExecuteTime": { - "end_time": "2025-11-10T21:10:51.298653Z", - "start_time": "2025-11-10T21:10:51.282654Z" - } - }, - "cell_type": "markdown", - "source": "## Bad Data Request Example", - "id": "5c3b637cf29f849f" - }, - { - "metadata": { - "ExecuteTime": { - "end_time": "2025-11-10T21:17:12.631455Z", - "start_time": "2025-11-10T21:17:12.280322Z" - } - }, - "cell_type": "code", - "source": [ - "data = onc.getScalardata({'locationCode':'BACVP',\n", - " 'deviceCategoryCode':'CTD',\n", - " 'dateFrom': '2009-08-15T00:00:00.000Z',\n", - " 'dateTo': '2009-08-17T23:59:59.999Z',\n", - " 'rowLimit': 10}, allPages = True)\n", - "\n", - "if isinstance(data, dict):\n", - " print(data)\n", - "else: # Assume it is a requests.Response object.\n", - " print(data.status_code)" - ], - "id": "d27aaa83f916109e", - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025-11-10T21:17:12.620Z | onc-service | INFO | Requested: https://data.oceannetworks.ca/api/scalardata?locationCode=BACVP&deviceCategoryCode=CTD&dateFrom=2009-08-15T00%3A00%3A00.000Z&dateTo=2009-08-17T23%3A59%3A59.999Z&rowLimit=10&method=getByLocation&token=REDACTED\n", - "2025-11-10T21:17:12.620Z | onc-service | DEBUG | Response received in 0.341 seconds.\n", - "2025-11-10T21:17:12.620Z | onc-service | ERROR | HTTP Response: Bad Request (400)\n", - "2025-11-10T21:17:12.620Z | onc-service | ERROR | (API Error Code 127) A device with category CTD was deployed at location BACVP but not during the provided time range (20090815T000000.000Z to 20090817T235959.999Z). The deployment service can be used to determine a valid time range: https://data.oceannetworks.ca/api/deployments?locationCode=BACVP&deviceCategoryCode=CTD&token=REDACTED for query parameter(s) 'locationCode, deviceCategoryCode, dateFrom, dateTo'.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "400\n" - ] - } - ], - "execution_count": 6 - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 2 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.6" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/src/onc/modules/_DataProductFile.py b/src/onc/modules/_DataProductFile.py index b864080..16b11fa 100644 --- a/src/onc/modules/_DataProductFile.py +++ b/src/onc/modules/_DataProductFile.py @@ -19,11 +19,19 @@ class _DataProductFile: Is able to poll and wait if required """ - def __init__(self, dpRunId: int, index: str, baseUrl: str, token: str, verbosity: str = "INFO"): + def __init__( + self, + dpRunId: int, + index: str, + baseUrl: str, + token: str, + verbosity: str = "INFO", + ): self.verbosity = verbosity # Use child logger 'onc.poll' consistent with _PollLog from ._Messages import setup_logger - self._log = setup_logger('onc.poll', verbosity) + + self._log = setup_logger("onc.poll", verbosity) self._retries = 0 self._status = 202 self._downloaded = False @@ -78,7 +86,9 @@ def download( response, outPath, filename, overwrite ) except FileExistsError: - self._log.info(f' Skipping "{self._filePath}": File already exists.') + self._log.info( + f' Skipping "{self._filePath}": File already exists.' + ) self._status = 777 elif self._status == 202: # Still processing, wait and retry diff --git a/src/onc/modules/_Messages.py b/src/onc/modules/_Messages.py index 9957be4..821625c 100644 --- a/src/onc/modules/_Messages.py +++ b/src/onc/modules/_Messages.py @@ -1,22 +1,24 @@ import logging import re -import requests import time -REQ_MSG = "Requested: {}" # get request url -RESPONSE_TIME_MSG = "Response received in {} seconds." # requests.elapsed value. -RESPONSE_MSG = "HTTP Response: {} ({})" # Brief description, status code -MULTIPAGE_MSG = ("The requested data quantity is greater than the " - "supplied row limit and will be downloaded over multiple requests.") +import requests +REQ_MSG = "Requested: {}" # get request url +RESPONSE_TIME_MSG = "Response received in {} seconds." # requests.elapsed value. +RESPONSE_MSG = "HTTP Response: {} ({})" # Brief description, status code +MULTIPAGE_MSG = ( + "The requested data quantity is greater than the " + "supplied row limit and will be downloaded over multiple requests." +) LEVEL_MAP = { - 'CRITICAL': logging.CRITICAL, - 'ERROR': logging.ERROR, - 'WARNING': logging.WARNING, - 'INFO': logging.INFO, - 'DEBUG': logging.DEBUG, + "CRITICAL": logging.CRITICAL, + "ERROR": logging.ERROR, + "WARNING": logging.WARNING, + "INFO": logging.INFO, + "DEBUG": logging.DEBUG, } @@ -24,20 +26,28 @@ class OnclibFormatter(logging.Formatter): """ Custom formatter that removes prefix for INFO level logs. """ + def format(self, record): if record.levelno == logging.INFO: return record.getMessage() return super().format(record) -def setup_logger(logger_name: str = 'onc', - level: int | str = 'INFO') -> logging.Logger: +def setup_logger(logger_name: str = "onc", level: int | str = "INFO") -> logging.Logger: """ Set up a logger object for displaying verbose messages to console. - :param logger_name: The unique logger name to use. Can be shared between modules - :param level: The logging level to use. Default is 'INFO'. - :return: The configured logging.Logger object. + Parameters + ---------- + logger_name : str, optional + The unique logger name to use. Can be shared between modules. + level : int or str, optional + The logging level to use. Default is 'INFO'. + + Returns + ------- + logging.Logger + The configured logging.Logger object. """ # Ensure level is a valid logging level integer @@ -55,8 +65,10 @@ def setup_logger(logger_name: str = 'onc', console.setLevel(level) # Set the logging format. - dtfmt = '%Y-%m-%dT%H:%M:%S' - strfmt = f'%(asctime)s.%(msecs)03dZ | %(name)-12s | %(levelname)-8s | %(message)s' + dtfmt = "%Y-%m-%dT%H:%M:%S" + strfmt = ( + "%(asctime)s.%(msecs)03dZ | %(name)-12s | %(levelname)-8s | %(message)s" + ) fmt = OnclibFormatter(strfmt, datefmt=dtfmt) fmt.converter = time.gmtime @@ -70,15 +82,21 @@ def setup_logger(logger_name: str = 'onc', return logger - def scrub_token(input: str) -> str: """ Replace a token in a query URL or other string with the string 'REDACTED' so that users don't accidentally commit their tokens to public repositories if ONC Info/Warnings are too verbose. - :param input: An Oceans 3.0 API URL or string with a token query parameter. - :return: A scrubbed url. + Parameters + ---------- + input : str + An Oceans 3.0 API URL or string with a token query parameter. + + Returns + ------- + str + A scrubbed url. """ return re.sub(r"([?&]token=)[a-f0-9-]{36}", r"\1REDACTED", input) @@ -87,9 +105,17 @@ def build_error_message(response: requests.Response, redact_token: bool) -> str: """ Build an error message from a requests.Response object. - :param response: A requests.Response object. - :param redact_token: If true, redact tokens before returning an error message. - :return: An error message. + Parameters + ---------- + response : requests.Response + A requests.Response object. + redact_token : bool + If true, redact tokens before returning an error message. + + Returns + ------- + str + An error message. """ payload = response.json() message = payload.get("message") @@ -98,15 +124,16 @@ def build_error_message(response: requests.Response, redact_token: bool) -> str: errors = payload["errors"] error_messages = [] for error in errors: - emsg = (f"(API Error Code {error['errorCode']}) " - f"{error['errorMessage']} for query parameter(s) " - f"'{error['parameter']}'.") + emsg = ( + f"(API Error Code {error['errorCode']}) " + f"{error['errorMessage']} for query parameter(s) " + f"'{error['parameter']}'." + ) error_messages.append(emsg) - error_message = '\n'.join(error_messages) + error_message = "\n".join(error_messages) else: error_message = None - msg = '\n'.join([m for m in (message, error_message) if m is not None]) - if redact_token is True and 'token=' in msg: + msg = "\n".join([m for m in (message, error_message) if m is not None]) + if redact_token is True and "token=" in msg: msg = scrub_token(msg) return msg - diff --git a/src/onc/modules/_MultiPage.py b/src/onc/modules/_MultiPage.py index 61a7ec6..af458de 100644 --- a/src/onc/modules/_MultiPage.py +++ b/src/onc/modules/_MultiPage.py @@ -4,31 +4,25 @@ from time import time import dateutil.parser -import humanize +from onc.modules._Messages import ( + setup_logger, +) from ._util import _formatDuration -from onc.modules._Messages import (setup_logger, MULTIPAGE_MSG, - build_error_message, - scrub_token, - REQ_MSG, - RESPONSE_TIME_MSG, - RESPONSE_MSG) - - # Handles data multi-page downloads (scalardata, rawdata, archivefiles) class _MultiPage: def __init__(self, parent: object): self.parent = weakref.ref(parent) self.result = None - self._log = setup_logger('onc.multi', self._config('verbosity') or 'INFO') + self._log = setup_logger("onc.multi", self._config("verbosity") or "INFO") def _config(self, key): p = self.parent() if p is None: return None - if hasattr(p, '_config'): + if hasattr(p, "_config"): return p._config(key) return getattr(p, key, None) @@ -36,7 +30,11 @@ def getAllPages(self, service: str, url: str, filters: dict): """ Requests all pages from the service, with the url and filters Multiple pages will be downloaded until completed - @return: Service response with concatenated data for all pages obtained + + Returns + ------- + dict + Service response with concatenated data for all pages obtained. """ # pop archivefiles extension extension = None @@ -48,28 +46,38 @@ def getAllPages(self, service: str, url: str, filters: dict): start = time() response, responseTime = self._doPageRequest(url, filters, service, extension) - if isinstance(response,dict): + if isinstance(response, dict): rNext = response["next"] if rNext is not None: - self._log.info("The requested data quantity is greater than the supplied " - "row limit and will be downloaded over multiple requests.") + self._log.info( + "The requested data quantity is greater than the supplied " + "row limit and will be downloaded over multiple requests." + ) pageCount = 1 pageEstimate = self._estimatePages(response, service) if pageEstimate > 0: # Exclude the first page when calculating the time estimation timeEstimate = _formatDuration((pageEstimate - 1) * responseTime) - self._log.info(f'Download time for page {pageCount}: {round(responseTime,2)} seconds') - self._log.info(f'Est. number of pages remaining for download: {pageEstimate-1}') - self._log.info(f'Est. number of seconds to download remaining data: {timeEstimate}') + self._log.info( + f"Download time for page {pageCount}: {round(responseTime, 2)} seconds" + ) + self._log.info( + f"Est. number of pages remaining for download: {pageEstimate - 1}" + ) + self._log.info( + f"Est. number of seconds to download remaining data: {timeEstimate}" + ) # keep downloading pages until next is None while rNext is not None: pageCount += 1 rowCount = self._rowCount(response, service) - self._log.info(f" Submitting request for page {pageCount} ({rowCount} samples)...") + self._log.info( + f" Submitting request for page {pageCount} ({rowCount} samples)..." + ) nextResponse, nextTime = self._doPageRequest( url, rNext["parameters"], service, extension @@ -81,7 +89,9 @@ def getAllPages(self, service: str, url: str, filters: dict): totalTime = _formatDuration(time() - start) - self._log.info(f" Downloaded {self._rowCount(response, service):d} total samples in {totalTime}.") + self._log.info( + f" Downloaded {self._rowCount(response, service):d} total samples in {totalTime}." + ) response["next"] = None return response @@ -92,8 +102,22 @@ def _doPageRequest( """ Wraps the _doRequest method Performs additional processing of the response for certain services - @param extension: Only provide for archivefiles filtering - Returns a tuple (jsonResponse, duration) + + Parameters + ---------- + url : str + API endpoint URL. + filters : dict + Filters for the request. + service : str + Name of the service (e.g. archivefile). + extension : str, optional + Only provide for archivefiles filtering. + + Returns + ------- + tuple + (jsonResponse, duration) """ if service.startswith("archivefile"): response, duration = self.parent()._doRequest(url, filters, getTime=True) diff --git a/src/onc/modules/_OncArchive.py b/src/onc/modules/_OncArchive.py index 342a2c6..759e061 100644 --- a/src/onc/modules/_OncArchive.py +++ b/src/onc/modules/_OncArchive.py @@ -6,7 +6,7 @@ from ._MultiPage import _MultiPage from ._OncService import _OncService -from ._util import _createErrorMessage, _formatDuration, saveAsFile +from ._util import _formatDuration, saveAsFile class _OncArchive(_OncService): @@ -17,7 +17,6 @@ class _OncArchive(_OncService): def __init__(self, parent: object): super().__init__(parent) - def getArchivefileByLocation(self, filters: dict, allPages: bool): """ Return a list of archived files for a device category in a location. @@ -75,6 +74,7 @@ def downloadArchivefile(self, filename: str = "", overwrite: bool = False): else: from ._Messages import build_error_message + self._log.error(build_error_message(response, self._config("redact_token"))) response.raise_for_status() diff --git a/src/onc/modules/_OncDelivery.py b/src/onc/modules/_OncDelivery.py index b242b2b..3889b94 100644 --- a/src/onc/modules/_OncDelivery.py +++ b/src/onc/modules/_OncDelivery.py @@ -8,7 +8,7 @@ from ._DataProductFile import _DataProductFile from ._OncService import _OncService from ._PollLog import _PollLog -from ._util import _createErrorMessage, _formatSize +from ._util import _formatSize class _OncDelivery(_OncService): @@ -111,7 +111,10 @@ def runDataProduct(self, dpRequestId: int, waitComplete: bool): data = response.json() else: from ._Messages import build_error_message - self._log.error(build_error_message(response, self._config("redact_token"))) + + self._log.error( + build_error_message(response, self._config("redact_token")) + ) response.raise_for_status() if waitComplete: @@ -191,7 +194,9 @@ def _downloadProductFiles( timeout = self._config("timeout") self._log.info(f"Downloading data product files with runId {runId}...") - dpf = _DataProductFile(runId, str(index), baseUrl, token, self._config("verbosity")) + dpf = _DataProductFile( + runId, str(index), baseUrl, token, self._config("verbosity") + ) # loop thorugh file indexes while doLoop: @@ -208,7 +213,9 @@ def _downloadProductFiles( # file was downloaded (200), or skipped before downloading (777) fileList.append(dpf.getInfo()) index += 1 - dpf = _DataProductFile(runId, str(index), baseUrl, token, self._config("verbosity")) + dpf = _DataProductFile( + runId, str(index), baseUrl, token, self._config("verbosity") + ) elif status != 202 or (fileCount > 0 and index >= fileCount): # no more files to download @@ -216,7 +223,9 @@ def _downloadProductFiles( # get metadata if required if getMetadata: - dpf = _DataProductFile(runId, "meta", baseUrl, token, self._config("verbosity")) + dpf = _DataProductFile( + runId, "meta", baseUrl, token, self._config("verbosity") + ) try: status = dpf.download( timeout, diff --git a/src/onc/modules/_OncDiscovery.py b/src/onc/modules/_OncDiscovery.py index eb0fd74..8ae32f8 100644 --- a/src/onc/modules/_OncDiscovery.py +++ b/src/onc/modules/_OncDiscovery.py @@ -10,7 +10,6 @@ class _OncDiscovery(_OncService): def __init__(self, parent: object): super().__init__(parent) - def _discoveryRequest(self, filters: dict, service: str): url = self._serviceUrl(service) filters["token"] = self._config("token") diff --git a/src/onc/modules/_OncService.py b/src/onc/modules/_OncService.py index 4f156a0..39392dc 100644 --- a/src/onc/modules/_OncService.py +++ b/src/onc/modules/_OncService.py @@ -1,19 +1,14 @@ -import logging import weakref -from time import time -from urllib import parse import requests - -from ._util import _createErrorMessage, _formatDuration -from onc.modules._Messages import (setup_logger, - build_error_message, - scrub_token, - REQ_MSG, - RESPONSE_TIME_MSG, - RESPONSE_MSG) - - +from onc.modules._Messages import ( + REQ_MSG, + RESPONSE_MSG, + RESPONSE_TIME_MSG, + build_error_message, + scrub_token, + setup_logger, +) class _OncService: @@ -23,7 +18,7 @@ class _OncService: def __init__(self, parent: object): self.parent = weakref.ref(parent) - self._log = setup_logger('onc.service', level=self._config('verbosity')) + self._log = setup_logger("onc.service", level=self._config("verbosity")) def _doRequest(self, url: str, filters: dict | None = None, getTime: bool = False): """ @@ -53,10 +48,7 @@ def _doRequest(self, url: str, filters: dict | None = None, getTime: bool = Fals response = requests.get(url, filters, timeout=timeout) if self._config("redact_token"): - try: - response_url = scrub_token(response.url) - except: - response_url = response.url + response_url = scrub_token(response.url) else: response_url = response.url @@ -65,7 +57,7 @@ def _doRequest(self, url: str, filters: dict | None = None, getTime: bool = Fals # Display the time it took for ONC to respond in seconds. # The requests.Response.elapsed value is a datetime.timedelta object. - responseTime = round(response.elapsed.total_seconds(),3) # To milliseconds. + responseTime = round(response.elapsed.total_seconds(), 3) # To milliseconds. self._log.debug(RESPONSE_TIME_MSG.format(responseTime)) json_response = response.json() @@ -83,8 +75,7 @@ def _doRequest(self, url: str, filters: dict | None = None, getTime: bool = Fals else: self._log.error(RESPONSE_MSG.format(response.reason, response.status_code)) - self._log.error(build_error_message(response, - self._config("redact_token"))) + self._log.error(build_error_message(response, self._config("redact_token"))) if self._config("raise_http_errors"): response.raise_for_status() @@ -93,7 +84,6 @@ def _doRequest(self, url: str, filters: dict | None = None, getTime: bool = Fals return response, responseTime return response - def _serviceUrl(self, service: str): """ Returns the absolute url for a given ONC API service @@ -144,4 +134,3 @@ def _delegateByFilters(self, byDevice, byLocation, **kwargs): "'locationCode' and 'deviceCategoryCode', " "or a 'deviceCode' present." ) - diff --git a/src/onc/modules/_PollLog.py b/src/onc/modules/_PollLog.py index 2de876f..e7595cb 100644 --- a/src/onc/modules/_PollLog.py +++ b/src/onc/modules/_PollLog.py @@ -1,5 +1,6 @@ from onc.modules._Messages import setup_logger + class _PollLog: """ A helper for DataProductFile @@ -8,13 +9,16 @@ class _PollLog: def __init__(self, verbosity: str): """ - @param verbosity standard logging level string + Parameters + ---------- + verbosity : str + Standard logging level string. """ self._messages = [] # unique messages returned during the product order self._runStart = 0.0 # {float} timestamp (seconds) self._runEnd = 0.0 self._verbosity = verbosity - self._log = setup_logger('onc.poll', level=verbosity) + self._log = setup_logger("onc.poll", level=verbosity) self._doLogFileCount = True def logMessage(self, response): @@ -38,7 +42,9 @@ def logMessage(self, response): if origin == "run": fileCount = response[0]["fileCount"] if self._doLogFileCount and fileCount > 0: - self._log.info(f" {fileCount} files generated for this data product") + self._log.info( + f" {fileCount} files generated for this data product" + ) self._doLogFileCount = False self._messages.append(msg) diff --git a/src/onc/modules/_util.py b/src/onc/modules/_util.py index e867205..87bae0e 100644 --- a/src/onc/modules/_util.py +++ b/src/onc/modules/_util.py @@ -35,7 +35,11 @@ def saveAsFile( def _formatSize(size: float) -> str: """ Returns a formatted file size string representation - @param size: {float} Size in bytes + + Parameters + ---------- + size : float + Size in bytes. """ return humanize.naturalsize(size) @@ -43,7 +47,11 @@ def _formatSize(size: float) -> str: def _formatDuration(secs: float) -> str: """ Returns a formatted time duration string representation of a duration in seconds - @param seconds: float + + Parameters + ---------- + secs : float + Duration in seconds. """ if secs < 1.0: txtDownTime = f"{secs:.3f} seconds" @@ -76,7 +84,6 @@ def _createErrorMessage(response: requests.Response) -> str: elif status == 401: return ( f"Status 401 - Unauthorized: {response.url}\n" - "Please check that your Web Services API token is valid. " "Find your token in your registered profile at " "https://data.oceannetworks.ca." diff --git a/src/onc/onc.py b/src/onc/onc.py index 32695b2..2950019 100644 --- a/src/onc/onc.py +++ b/src/onc/onc.py @@ -6,12 +6,12 @@ from pathlib import Path from dateutil import parser +from onc.modules._Messages import setup_logger from onc.modules._OncArchive import _OncArchive from onc.modules._OncDelivery import _OncDelivery from onc.modules._OncDiscovery import _OncDiscovery from onc.modules._OncRealTime import _OncRealTime -from onc.modules._Messages import setup_logger class ONC: """ @@ -62,6 +62,7 @@ def __init__( ): if "showInfo" in kwargs or "showWarning" in kwargs: import warnings + warnings.warn( "showInfo and showWarning are deprecated. Use verbosity instead.", DeprecationWarning, @@ -84,7 +85,7 @@ def __init__( self.verbosity = verbosity self.redact_token = redact_token self.raise_http_errors = raise_http_errors - self._log = setup_logger('onc', self.verbosity) + self._log = setup_logger("onc", self.verbosity) self.token = re.sub(r"[^a-zA-Z0-9\-]+", "", token) diff --git a/src/onc/util/util.py b/src/onc/util/util.py index b66b5e8..f407837 100644 --- a/src/onc/util/util.py +++ b/src/onc/util/util.py @@ -341,9 +341,14 @@ def copyFieldIfExists(fromDic, toDic, keys): """ Copy the field at name from fromDic to toDic only if it exists - @param fromDic: Origin Dictionary - @param toDic: Destination Dictionary - @param keys: Array of keys of the elements to copy + Parameters + ---------- + fromDic : dict + Origin Dictionary. + toDic : dict + Destination Dictionary. + keys : list + Array of keys of the elements to copy. """ for key in keys: if key in fromDic: diff --git a/tests/conftest.py b/tests/conftest.py index cc87c02..3cdb286 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,7 +1,8 @@ import os from pathlib import Path -import requests + import pytest +import requests from dotenv import load_dotenv from onc import ONC @@ -18,6 +19,7 @@ def pytest_configure(): def requester(tmp_path) -> ONC: return ONC(production=is_prod, outPath=tmp_path) + @pytest.fixture def err_400(): return f"{requests.codes.bad} Client Error"