From 876b9a86315db6d0cd00e453f7f3793b64f2400f Mon Sep 17 00:00:00 2001
From: Pablo Boizeau <pablo.boizeau@ird.fr>
Date: Tue, 10 Dec 2024 14:15:57 +0100
Subject: [PATCH 1/6] Redirect logger to stdout
---
theia_dumper/logger.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/theia_dumper/logger.py b/theia_dumper/logger.py
index 0021812..c335763 100644
--- a/theia_dumper/logger.py
+++ b/theia_dumper/logger.py
@@ -1,4 +1,5 @@
"""Logging stuff."""
import logging
-logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.DEBUG)
+import sys
+logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.DEBUG, stream=sys.stdout)
logger = logging.getLogger(__name__)
--
GitLab
From 6d2afbad6ab9b922c08fe770d79f2cae6d6000b6 Mon Sep 17 00:00:00 2001
From: Pablo Boizeau <pablo.boizeau@ird.fr>
Date: Tue, 10 Dec 2024 14:16:58 +0100
Subject: [PATCH 2/6] Improve dinamis post error retrieval
---
theia_dumper/stac.py | 6 +++++-
1 file changed, 5 insertions(+), 1 deletion(-)
diff --git a/theia_dumper/stac.py b/theia_dumper/stac.py
index 41a7809..9f5ea4f 100644
--- a/theia_dumper/stac.py
+++ b/theia_dumper/stac.py
@@ -55,7 +55,11 @@ def post_or_put(url: str, data: dict):
if not resp.status_code == 404:
resp.raise_for_status()
else:
- resp.raise_for_status()
+ try:
+ resp.raise_for_status()
+ except Exception as e:
+ logger.error(eval(resp.content)["detail"])
+ raise(e)
def load(obj_pth):
--
GitLab
From 793db1e87999896b431ce63d65cb59bc5b98d09e Mon Sep 17 00:00:00 2001
From: Pablo Boizeau <pablo.boizeau@ird.fr>
Date: Tue, 10 Dec 2024 14:48:13 +0100
Subject: [PATCH 3/6] Fix logging loglevel from environment
---
theia_dumper/logger.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/theia_dumper/logger.py b/theia_dumper/logger.py
index c335763..9a23f90 100644
--- a/theia_dumper/logger.py
+++ b/theia_dumper/logger.py
@@ -1,5 +1,6 @@
"""Logging stuff."""
import logging
import sys
-logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.DEBUG, stream=sys.stdout)
+import os
+logging.basicConfig(format='%(levelname)s:%(message)s', level=os.environ.get("LOGLEVEL", "INFO"), stream=sys.stdout)
logger = logging.getLogger(__name__)
--
GitLab
From 128091e2c94638f1d454768c24413d3888952ede Mon Sep 17 00:00:00 2001
From: Pablo Boizeau <pablo.boizeau@ird.fr>
Date: Tue, 10 Dec 2024 14:48:35 +0100
Subject: [PATCH 4/6] Improve items publish logic and fix asset href
---
theia_dumper/stac.py | 47 +++++++++++++++++++++++---------------------
1 file changed, 25 insertions(+), 22 deletions(-)
diff --git a/theia_dumper/stac.py b/theia_dumper/stac.py
index 9f5ea4f..4892949 100644
--- a/theia_dumper/stac.py
+++ b/theia_dumper/stac.py
@@ -139,10 +139,10 @@ class TransactionsHandler:
self.storage_bucket,
col_id
)
- local_assets_files = [asset.href for asset in item.assets.values()]
# Upload assets files
- for local_filename in local_assets_files:
+ for asset_name, asset in item.assets.items():
+ local_filename = asset.href
logger.debug("Local file: %s", local_filename)
target_url = local_filename.replace(
assets_root_dir,
@@ -165,29 +165,32 @@ class TransactionsHandler:
# Upload file
logger.info("Uploading %s ...", local_filename)
- dinamis_sdk.push(
- local_filename=local_filename,
- target_url=target_url
- )
+ try:
+ dinamis_sdk.push(
+ local_filename=local_filename,
+ target_url=target_url
+ )
+ except Exception as e:
+ logger.error(e)
+ raise(e)
# Update assets hrefs
logger.debug("Updating assets HREFs ...")
- for asset_name in item.assets:
- item.assets[asset_name].href = target_url
-
- # Push item
- logger.info(
- "Publishing item \"%s\" in collection \"%s\"",
- item.id,
- col_id
- )
- post_or_put(
- urljoin(
- self.stac_endpoint,
- f"collections/{col_id}/items"
- ),
- item.to_dict(transform_hrefs=False)
- )
+ asset.href = target_url
+
+ # Push item
+ logger.info(
+ "Publishing item \"%s\" in collection \"%s\"",
+ item.id,
+ col_id
+ )
+ post_or_put(
+ urljoin(
+ self.stac_endpoint,
+ f"collections/{col_id}/items"
+ ),
+ item.to_dict(transform_hrefs=False)
+ )
def publish_items(self, items: List[Item]):
"""Publish items."""
--
GitLab
From 54bcd2cc9ef3925c9286a8ffd0c00e0f61fbf803 Mon Sep 17 00:00:00 2001
From: Pablo Boizeau <pablo.boizeau@ird.fr>
Date: Tue, 10 Dec 2024 15:20:44 +0100
Subject: [PATCH 5/6] Change bucket used in CI
---
tests/all.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/tests/all.py b/tests/all.py
index bbe2590..4fcedd2 100644
--- a/tests/all.py
+++ b/tests/all.py
@@ -10,7 +10,7 @@ from theia_dumper import stac
handler = stac.TransactionsHandler(
stac_endpoint="https://stacapi-cdos.apps.okd.crocc.meso.umontpellier.fr",
storage_endpoint="https://s3-data.meso.umontpellier.fr",
- storage_bucket="sm1-gdc",
+ storage_bucket="sm1-gdc-tests",
assets_overwrite=True
)
--
GitLab
From b67d6b85eeb432ae867762e5c1c261ce1bb9f0cd Mon Sep 17 00:00:00 2001
From: Pablo Boizeau <pablo.boizeau@ird.fr>
Date: Tue, 10 Dec 2024 15:49:01 +0100
Subject: [PATCH 6/6] Set dinamis_sdk version and increase version
---
pyproject.toml | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/pyproject.toml b/pyproject.toml
index 5120816..553cfe9 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
[project]
name = "theia_dumper"
-version = "0.0.1"
+version = "0.0.2"
description = "THEIA-MTP geospatial data publisher"
authors = [
{ name = "Rémi Cresson", email = "remi.cresson@inrae.fr" },
@@ -13,7 +13,7 @@ requires-python = ">=3.9"
dependencies = [
"pystac",
"pystac_client",
- "dinamis_sdk > 0.3.0",
+ "dinamis_sdk==0.3.3",
"requests",
"rio-cogeo"
]
--
GitLab