Skip to content

Commit

Permalink
update logging
Browse files Browse the repository at this point in the history
  • Loading branch information
kai-tub committed Jun 21, 2024
1 parent d4be93c commit 777d369
Showing 1 changed file with 2 additions and 6 deletions.
8 changes: 2 additions & 6 deletions run.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,18 +21,14 @@

def read_single_band_raster(path):
with rasterio.open(path) as r:
d = r.read(1)
print(d.shape)
return d
return r.read(1)

def s2_safetensor_generator(lmdb_key: str, files: list[Path]) -> bytes:
# In Python the dictionary insertion order is stable!
# order the data here to make it clear that we are doing it
# to order the safetensor entries!
files = sorted(files, key=lambda f: BIGEARTHNET_S2_ORDERING.index(f.stem[-3:]))
data = {f.stem[-3:]: read_single_band_raster(f) for f in files}
print(data)
log.debug(f"Data Keys: {data.keys()}")
return save(data, metadata=None)


Expand Down Expand Up @@ -100,8 +96,8 @@ def main(
log.debug("About to serialize data in chunks")
for keys_chunk in tqdm(chunked(lmdb_keys, 512)):
with env.begin(write=True) as txn:
log.debug(f"First key of the chunk is: {keys_chunk[0]}")
for key in keys_chunk:
log.debug(f"writing key: {key}")
if not txn.put(str(key).encode(), s2_safetensor_generator(key, grouped[key]), overwrite=False):
sys.exit("Program is overwriting data in the DB! This should never happen!")

Expand Down

0 comments on commit 777d369

Please sign in to comment.