Skip to content

Commit

Permalink
refactor: Merge branch 'dev' into refac_swap
Browse files Browse the repository at this point in the history
  • Loading branch information
@lsetiawan
lsetiawan committed Oct 9, 2023
2 parents 617f510 + 0bc534e commit 86bcab3
Show file tree
Hide file tree
Showing 5 changed files with 148 additions and 119 deletions.
37 changes: 11 additions & 26 deletions .github/workflows/build.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,25 +9,20 @@ on:
workflow_dispatch:

env:
CONDA_ENV: echopype
NUM_WORKERS: 2

jobs:
test:
name: ${{ matrix.python-version }}-build
runs-on: ubuntu-20.04
runs-on: ubuntu-latest
if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }}
continue-on-error: ${{ matrix.experimental }}
strategy:
fail-fast: false
matrix:
python-version: ["3.9", "3.10"] # TODO: add back 3.11 once parsed2zarr is fixed
python-version: ["3.9", "3.10", "3.11"]
runs-on: [ubuntu-latest]
experimental: [false]
include:
- runs-on: ubuntu-latest
python-version: "3.11"
experimental: true
services:
# TODO: figure out how to update tag when there's a new one
minio:
Expand All @@ -46,30 +41,20 @@ jobs:
- name: Set environment variables
run: |
echo "PYTHON_VERSION=${{ matrix.python-version }}" >> $GITHUB_ENV
- name: Setup micromamba
uses: mamba-org/setup-micromamba@v1
- name: Set up Python
uses: actions/setup-python@v4.7.1
with:
environment-file: .ci_helpers/py${{ matrix.python-version }}.yaml
environment-name: ${{ env.CONDA_ENV }}
cache-environment: true
post-cleanup: 'all'
- name: Print conda environment
shell: bash -l {0}
run: |
micromamba info
micromamba list
python-version: ${{ matrix.python-version }}
- name: Upgrade pip
run: python -m pip install --upgrade pip
- name: Remove docker-compose python
if: ${{ matrix.python-version == '3.10' || matrix.python-version == '3.11' }}
shell: bash -l {0}
run: sed -i "/docker-compose/d" requirements-dev.txt
- name: Install dev tools
shell: bash -l {0}
run: |
micromamba install -c conda-forge -n ${{ env.CONDA_ENV }} --yes --file requirements-dev.txt
run: python -m pip install -r requirements-dev.txt
- name: Install echopype
shell: bash -l {0}
run: |
python -m pip install -e .[plot]
run: python -m pip install -e ".[plot]"
- name: Print installed packages
run: python -m pip list
- name: Copying test data to services
shell: bash -l {0}
run: |
Expand Down
39 changes: 12 additions & 27 deletions .github/workflows/pr.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ on:
paths-ignore: ["**/docker.yaml", "docs"]

env:
CONDA_ENV: echopype
NUM_WORKERS: 2

jobs:
Expand All @@ -17,16 +16,9 @@ jobs:
strategy:
fail-fast: false
matrix:
python-version: ["3.9", "3.10"] # TODO: add back 3.11 once parsed2zarr is fixed
python-version: ["3.9", "3.10", "3.11"]
runs-on: [ubuntu-latest]
experimental: [false]
include:
- runs-on: ubuntu-latest
python-version: "3.11"
experimental: true
defaults:
run:
shell: bash -l {0}
services:
# TODO: figure out how to update tag when there's a new one
minio:
Expand All @@ -42,35 +34,28 @@ jobs:
uses: actions/checkout@v4
with:
fetch-depth: 0 # Fetch all history for all branches and tags.
- name: Set up Python
uses: actions/setup-python@v4.7.1
with:
python-version: ${{ matrix.python-version }}
- name: Upgrade pip
run: python -m pip install --upgrade pip
- name: Set environment variables
run: |
echo "PYTHON_VERSION=${{ matrix.python-version }}" >> $GITHUB_ENV
- name: Setup micromamba
uses: mamba-org/setup-micromamba@v1
with:
environment-file: .ci_helpers/py${{ matrix.python-version }}.yaml
environment-name: ${{ env.CONDA_ENV }}
cache-environment: true
post-cleanup: 'all'
- name: Print conda environment
run: |
micromamba info
micromamba list
- name: Remove docker-compose python
if: ${{ matrix.python-version == '3.10' || matrix.python-version == '3.11' }}
run: sed -i "/docker-compose/d" requirements-dev.txt
- name: Install dev tools
run: |
micromamba install -c conda-forge -n ${{ env.CONDA_ENV }} --yes --file requirements-dev.txt
run: python -m pip install -r requirements-dev.txt
# We only want to install this on one run, because otherwise we'll have
# duplicate annotations.
- name: Install error reporter
if: ${{ matrix.python-version == '3.9' }}
run: |
python -m pip install pytest-github-actions-annotate-failures
run: python -m pip install pytest-github-actions-annotate-failures
- name: Install echopype
run: |
python -m pip install -e .[plot]
run: python -m pip install -e ".[plot]"
- name: Print installed packages
run: python -m pip list
- name: Copying test data to services
run: |
python .ci_helpers/docker/setup-services.py --deploy --data-only --http-server ${{ job.services.httpserver.id }}
Expand Down
49 changes: 29 additions & 20 deletions echopype/convert/parse_azfp.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -91,26 +91,35 @@ def load_AZFP_xml(self):
"""

xmlmap = fsspec.get_mapper(self.xml_path, **self.storage_options)
root = ET.parse(xmlmap.fs.open(xmlmap.root)).getroot()

for child in root.iter():
if len(child.tag) > 3 and not child.tag.startswith("VTX"):
camel_case_tag = camelcase2snakecase(child.tag)
else:
camel_case_tag = child.tag
if len(child.attrib) > 0:
for key, val in child.attrib.items():
self.parameters[camel_case_tag + "_" + camelcase2snakecase(key)].append(val)

if all(char == "\n" for char in child.text):
continue
else:
try:
val = int(child.text)
except ValueError:
val = float(child.text)

self.parameters[camel_case_tag].append(val)
phase_number = None
for event, child in ET.iterparse(xmlmap.fs.open(xmlmap.root), events=("start", "end")):
if event == "end" and child.tag == "Phases":
phase_number = None
if event == "start":
if len(child.tag) > 3 and not child.tag.startswith("VTX"):
camel_case_tag = camelcase2snakecase(child.tag)
else:
camel_case_tag = child.tag

if len(child.attrib) > 0:
for key, val in child.attrib.items():
attrib_tag = camel_case_tag + "_" + camelcase2snakecase(key)
if phase_number is not None and camel_case_tag != "phase":
attrib_tag += f"_phase{phase_number}"
self.parameters[attrib_tag].append(val)
if child.tag == "Phase":
phase_number = val

if all(char == "\n" for char in child.text):
continue
else:
try:
val = int(child.text)
except ValueError:
val = float(child.text)
if phase_number is not None and camel_case_tag != "phase":
camel_case_tag += f"_phase{phase_number}"
self.parameters[camel_case_tag].append(val)

# Handling the case where there is only one value for each parameter
for key, val in self.parameters.items():
Expand Down
60 changes: 44 additions & 16 deletions echopype/convert/set_groups_azfp.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ def _create_unique_channel_name(self):
"""

serial_number = self.parser_obj.unpacked_data["serial_number"]
frequency_number = self.parser_obj.parameters["frequency_number"]
frequency_number = self.parser_obj.parameters["frequency_number_phase1"]

if serial_number.size == 1:
freq_as_str = self.freq_sorted.astype(int).astype(str)
Expand Down Expand Up @@ -500,7 +500,25 @@ def set_vendor(self) -> xr.Dataset:
unpacked_data = self.parser_obj.unpacked_data
parameters = self.parser_obj.parameters
ping_time = self.parser_obj.ping_time
tdn = parameters["pulse_len"][self.freq_ind_sorted] / 1e6
phase_params = ["burst_interval", "pings_per_burst", "average_burst_pings"]
phase_freq_params = [
"dig_rate",
"range_samples",
"range_averaging_samples",
"lock_out_index",
"gain",
"storage_format",
]
tdn = []
for num in parameters["phase_number"]:
tdn.append(parameters[f"pulse_len_phase{num}"][self.freq_ind_sorted] / 1e6)
tdn = np.array(tdn)
for param in phase_freq_params:
for num in parameters["phase_number"]:
parameters[param].append(parameters[f"{param}_phase{num}"][self.freq_ind_sorted])
for param in phase_params:
for num in parameters["phase_number"]:
parameters[param].append(parameters[f"{param}_phase{num}"])
anc = np.array(unpacked_data["ancillary"]) # convert to np array for easy slicing

# Build variables in the output xarray Dataset
Expand Down Expand Up @@ -639,13 +657,13 @@ def set_vendor(self) -> xr.Dataset:
),
# parameters with channel dimension from XML file
"XML_transmit_duration_nominal": (
["channel"],
["phase_number", "channel"],
tdn,
{"long_name": "(From XML file) Nominal bandwidth of transmitted pulse"},
), # tdn comes from parameters
"XML_gain_correction": (
["channel"],
parameters["gain"][self.freq_ind_sorted],
["phase_number", "channel"],
parameters["gain"],
{"long_name": "(From XML file) Gain correction"},
),
"instrument_type": parameters["instrument_type"][0],
Expand All @@ -660,17 +678,17 @@ def set_vendor(self) -> xr.Dataset:
"parameter_version": parameters["parameter_version"],
"configuration_version": parameters["configuration_version"],
"XML_digitization_rate": (
["channel"],
parameters["dig_rate"][self.freq_ind_sorted],
["phase_number", "channel"],
parameters["dig_rate"],
{
"long_name": "(From XML file) Number of samples per second in kHz that is "
"processed by the A/D converter when digitizing the returned acoustic "
"signal"
},
),
"XML_lockout_index": (
["channel"],
parameters["lock_out_index"][self.freq_ind_sorted],
["phase_number", "channel"],
parameters["lock_out_index"],
{
"long_name": "(From XML file) The distance, rounded to the nearest "
"Bin Size after the pulse is transmitted that over which AZFP will "
Expand Down Expand Up @@ -713,17 +731,17 @@ def set_vendor(self) -> xr.Dataset:
),
"Sv_offset": (["channel"], Sv_offset),
"number_of_samples_digitized_per_pings": (
["channel"],
parameters["range_samples"][self.freq_ind_sorted],
["phase_number", "channel"],
parameters["range_samples"],
),
"number_of_digitized_samples_averaged_per_pings": (
["channel"],
parameters["range_averaging_samples"][self.freq_ind_sorted],
["phase_number", "channel"],
parameters["range_averaging_samples"],
),
# parameters with dim len=0 from XML file
"XML_sensors_flag": parameters["sensors_flag"],
"XML_burst_interval": (
[],
["phase_number"],
parameters["burst_interval"],
{
"long_name": "Time in seconds between bursts or between pings if the burst "
Expand All @@ -732,8 +750,14 @@ def set_vendor(self) -> xr.Dataset:
),
"XML_sonar_serial_number": parameters["serial_number"],
"number_of_frequency": parameters["num_freq"],
"number_of_pings_per_burst": parameters["pings_per_burst"],
"average_burst_pings_flag": parameters["average_burst_pings"],
"number_of_pings_per_burst": (
["phase_number"],
parameters["pings_per_burst"],
),
"average_burst_pings_flag": (
["phase_number"],
parameters["average_burst_pings"],
),
# temperature coefficients from XML file
**{
f"temperature_k{var}": (
Expand Down Expand Up @@ -789,6 +813,10 @@ def set_vendor(self) -> xr.Dataset:
list(range(len(unpacked_data["ancillary"][0]))),
),
"ad_len": (["ad_len"], list(range(len(unpacked_data["ad"][0])))),
"phase_number": (
["phase_number"],
sorted([int(num) for num in parameters["phase_number"]]),
),
},
)
return set_time_encodings(ds)
Loading

0 comments on commit 86bcab3

Please sign in to comment.