From 99b20c9b8c33666fd796941da926781dbd268107 Mon Sep 17 00:00:00 2001 From: Dawid Kowalczyk Date: Thu, 21 Nov 2024 13:32:02 +0100 Subject: [PATCH 1/2] Update to 5.1.3 --- .gitignore | 1 + Makefile | 10 +- README.md | 24 +-- images/docker/Dockerfile | 10 +- images/iso/build_iso.sh | 143 ++++++++++++++---- images/iso/build_iso_helper | 29 ++-- images/iso/isolinux/grub.cfg | 41 +++-- images/iso/ona/configure.sh | 4 +- images/packer/ona-service.json | 2 +- package_builder.py | 4 +- .../obsrvbl-ona/system/obsrvbl_ona.sudoers | 2 +- .../system/supervisord/ona-eta-monitor.sh | 2 +- .../system/supervisord/ona-pdns-monitor.sh | 2 +- packaging/scripts/postinst_UbuntuXenial.sh | 29 ---- .../scripts/postinst_UbuntuXenialContainer.sh | 29 ---- raspberry_pi_guide.md | 2 +- .../ona_service/installation/postinst.py | 2 +- .../ona_service/installation/system_tools.py | 6 +- src/scripts/requirements.txt | 23 +-- 19 files changed, 217 insertions(+), 148 deletions(-) delete mode 100644 packaging/scripts/postinst_UbuntuXenial.sh delete mode 100644 packaging/scripts/postinst_UbuntuXenialContainer.sh diff --git a/.gitignore b/.gitignore index da5bd6c2..22ecd2a4 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ index.html .coverage images/iso/*.iso +images/iso/local_files/ packaging/output/ packaging/root/opt/obsrvbl-ona/netflow/ packaging/root/opt/obsrvbl-ona/ipfix/ diff --git a/Makefile b/Makefile index da2abad9..3749650e 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. ARCH ?= amd64 -VERSION := 5.1.2 +VERSION := 5.1.3 SCRIPTS_DIR := src/scripts uPNA_DIR := src/uPNA @@ -71,13 +71,13 @@ ona-service_RaspbianJessie_%.deb: mkdir -p $(dir $@) python package_builder.py $(notdir $*) ${VERSION} RaspbianJessie -ona-service_UbuntuXenial_%.deb: +ona-service_UbuntuNoble_%.deb: mkdir -p $(dir $@) - python package_builder.py $(notdir $*) ${VERSION} UbuntuXenial + python package_builder.py $(notdir $*) ${VERSION} UbuntuNoble -ona-service_UbuntuXenialContainer_%.deb: +ona-service_UbuntuNobleContainer_%.deb: mkdir -p $(dir $@) - python package_builder.py $(notdir $*) ${VERSION} UbuntuXenialContainer + python package_builder.py $(notdir $*) ${VERSION} UbuntuNobleContainer .PHONY: clean clean: diff --git a/README.md b/README.md index f9534a41..78bb0141 100644 --- a/README.md +++ b/README.md @@ -2,24 +2,28 @@ This repository is where the development of the Observable Networks Appliance (ONA) takes place. The ONA software is used to collect input data for Observable Networks' network security service. It can run on a variety of platforms, including embedded computers, physical servers, virtual machines, cloud servers, and Docker containers. -## Supported platforms +## Download -The following platforms are officially supported: +### ISO (fully supported and recommended): -* [Ubuntu 18.04 and later](https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.2/ona-service_UbuntuXenial_amd64.deb) -* [RHEL 7 and compatible](https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.2/ona-service_RHEL_7_x86_64.rpm) -* [RHEL 8 and compatible](https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.2/ona-service_RHEL_8_x86_64.rpm) -* [Raspberry Pi with Raspbian (ARMHF)](https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.2/ona-service_RaspbianJessie_armhf.deb) +* [Ubuntu 24.04](https://assets-production.obsrvbl.com/ona-packages/iso/ona-24.04.1-v5.1.3/ona-24.04.1-server-amd64.iso) + +### Package files for manual installation: + +* [Ubuntu 24.04 and later](https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.3/ona-service_UbuntuNoble_amd64.deb) +* [RHEL 7 and compatible](https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.3/ona-service_RHEL_7_x86_64.rpm) +* [RHEL 8 and compatible](https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.3/ona-service_RHEL_8_x86_64.rpm) +* [Raspberry Pi with Raspbian (ARMHF)](https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.3/ona-service_RaspbianJessie_armhf.deb) ([installation guide](raspberry_pi_guide.md)) -* [Raspberry Pi with Raspbian (ARM64)](https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.2/ona-service_RaspbianJessie_aarch64.deb) +* [Raspberry Pi with Raspbian (ARM64)](https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.3/ona-service_RaspbianJessie_aarch64.deb) ([installation guide](raspberry_pi_guide.md)) * [Docker](https://github.com/obsrvbl/ona/blob/master/images/docker/Dockerfile) -To install the latest version on 20.04 (recommended for physical and virtual machine installations): +To install the latest version on Ubuntu: ``` -$ wget https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.2/ona-service_UbuntuXenial_amd64.deb -$ sudo apt install ./ona-service_UbuntuXenial_amd64.deb +$ wget https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.3/ona-service_UbuntuNoble_amd64.deb +$ sudo apt install ./ona-service_UbuntuNoble_amd64.deb ``` To monitor NetFlow traffic, you'll also need to install tools from the [CERT NetSA Security Suite](https://tools.netsa.cert.org/): diff --git a/images/docker/Dockerfile b/images/docker/Dockerfile index d82e6955..073d50a6 100644 --- a/images/docker/Dockerfile +++ b/images/docker/Dockerfile @@ -48,15 +48,15 @@ RUN curl -L -O https://assets-production.obsrvbl.com/ona-packages/netsa/v0.1.27/ && rm -rf netsa-pkg.deb # Use local copy of ONA service package if needed -# COPY ona-service_UbuntuXenialContainer_amd64.deb ./ +# COPY ona-service_UbuntuNobleContainer_amd64.deb ./ # Install ONA service -RUN if [ ! -f ./ona-service_UbuntuXenialContainer_amd64.deb ] ;\ - then curl -L -O https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.2/ona-service_UbuntuXenialContainer_amd64.deb ;\ +RUN if [ ! -f ./ona-service_UbuntuNobleContainer_amd64.deb ] ;\ + then curl -L -O https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.3/ona-service_UbuntuNobleContainer_amd64.deb ;\ else echo "Use cached package" ;fi \ - && apt-get update && apt-get install --assume-yes --fix-missing ./ona-service_UbuntuXenialContainer_amd64.deb \ + && apt-get update && apt-get install --assume-yes --fix-missing ./ona-service_UbuntuNobleContainer_amd64.deb \ && rm -rf /var/lib/apt/lists/* \ - && rm -rf ona-service_UbuntuXenialContainer_amd64.deb + && rm -rf ona-service_UbuntuNobleContainer_amd64.deb # Switch to the unprivileged user, set some local configuration, and start. COPY run.sh /opt/obsrvbl-ona/run.sh diff --git a/images/iso/build_iso.sh b/images/iso/build_iso.sh index 504c2cd2..50e43efb 100755 --- a/images/iso/build_iso.sh +++ b/images/iso/build_iso.sh @@ -21,9 +21,9 @@ # wrong. # -RELEASE="${RELEASE:-20.04.1}" +RELEASE="${RELEASE:-24.04.1}" ARCH="${ARCH:-amd64}" -VARIANT="${VARIANT:-legacy}" +VARIANT="${VARIANT:-subiquity}" DIR=$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd) @@ -45,47 +45,138 @@ while getopts "f:a:r:" opt ; do ;; esac done +# Newly added + ubuntu_name="ubuntu-${RELEASE}-live-server-${ARCH}.iso" + ona_name="ona-${RELEASE}-server-${ARCH}.iso" + ubuntu_url="${url:-$($DIR/build_iso_helper $RELEASE $VARIANT)}" + +# ubuntu_name="ubuntu-24.04.1-live-server-amd64.iso" +# ona_name="ona-${RELEASE}-server-${ARCH}.iso" +ONA_URL="https://s3.amazonaws.com/onstatic/ona-service/master/" +if [ -n "$PUBLIC_ONA" ]; then + ONA_URL="https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.2/" +fi +# netsa_pkg_name="netsa-pkg.deb" +ona_pkg_name="ona-service_UbuntuNoble_amd64.deb" -ubuntu_name="ubuntu-${RELEASE}-server-${ARCH}.iso" -ona_name="ona-${RELEASE}-server-${ARCH}.iso" -ubuntu_url="${url:-$($DIR/build_iso_helper $RELEASE $VARIANT)}" test -n "$ubuntu_url" || fatal "failed getting Ubuntu ISO download URL" -ona_service_url="https://s3.amazonaws.com/onstatic/ona-service/master/ona-service_UbuntuXenial_amd64.deb" -netsa_pkg_url="https://assets-production.obsrvbl.com/ona-packages/netsa/v0.1.27/netsa-pkg.deb" + + ONA_URL="https://s3.amazonaws.com/onstatic/ona-service/master/" + if [ -n "$PUBLIC_ONA" ]; then + ONA_URL="https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.2/" + + fi + + #ona_service_url="${ONA_URL}ona-service_UbuntuNoble_amd64.deb" + ona_service_url="https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.3/ona-service_UbuntuNoble_amd64.deb" + netsa_pkg_url="https://assets-production.obsrvbl.com/ona-packages/netsa/v0.1.27/netsa-pkg.deb" + + shift $(($OPTIND-1)) test $EUID -ne 0 && sudo="sudo" -which mkisofs 1> /dev/null || fatal "missing mkisofs: $sudo apt-get install genisoimage" -which isohybrid 1> /dev/null || fatal "missing isohybrid: $sudo apt-get install syslinux-utils" [[ -d "$DIR" ]] || fatal # invalid directory -[[ -d "$DIR"/working && $(ls -A "$DIR"/working) ]] && fatal # working directory exists and is not empty [[ -d "$DIR"/working ]] || mkdir "$DIR"/working # working directory does not exist, so create it + +major_version=$(echo "$RELEASE" | cut -d '.' -f 1) + +# Check if the major version number is greater than 20 +if [ "$major_version" -gt 20 ]; then + which xorriso 1> /dev/null || fatal "missing xorriso: $sudo apt-get install xorriso -y" + NEW_FORMAT=true + BOOT_CAT="/boot.catalog" + EFI='/boot/grub/i386-pc/eltorito.img' + ELTORITO='/boot/grub/i386-pc/eltorito.img' +else + which mkisofs 1> /dev/null || fatal "missing mkisofs: $sudo apt-get install genisoimage" + which isohybrid 1> /dev/null || fatal "missing isohybrid: $sudo apt-get install syslinux-utils" + BOOT_CAT="isolinux/boot.cat" + EFI="isolinux/isolinux.bin" + ELTORITO="boot/grub/efi.img" +fi + ( set -e + if [ ! -e "/root/$ubuntu_name" ]; then + curl -L -o /root/${ubuntu_name} "${ubuntu_url}" + fi + cd "$DIR"/working - curl -L -o ${ubuntu_name} "${ubuntu_url}" + #[[ -d "$DIR/local_files/" ]] && cp "$DIR"/local_files/* . curl -L -o netsa-pkg.deb "${netsa_pkg_url}" - curl -L -o ona-service.deb "${ona_service_url}" + #curl -L -o "${ona_pkg_name}" "${ona_service_url}" + $sudo cp /obsrvbl/images/iso/ona-service_UbuntuNoble_amd64.deb /obsrvbl/images/iso/working/ + + + + + +$sudo apt-get -y update +# you can install packages here if you want + +PACKAGES="apt-transport-https iptables-persistent ipset libjansson4 libltdl7 liblzo2-2 libnet1 libyaml-0-2 nano ntp ntpdate snmp tcpdump net-tools libsnappy1v5 python3-dateutil" +$sudo apt-get -yyqq install --download-only ${PACKAGES} + + + + + # local is root dir in ISO mkdir cdrom local - $sudo mount -o loop --read-only "${ubuntu_name}" cdrom + pwd + + $sudo mount -o loop --read-only "/root/${ubuntu_name}" cdrom rsync -av --quiet cdrom/ local - $sudo cp ../preseed/* local/preseed/ + + $sudo cp -r /var/cache/apt local $sudo cp -r ../ona local $sudo cp netsa-pkg.deb local/ona/netsa-pkg.deb - $sudo cp ona-service.deb local/ona/ona-service.deb - $sudo cp ../isolinux/txt.cfg local/isolinux/txt.cfg - $sudo cp ../isolinux/grub.cfg local/boot/grub/grub.cfg - $sudo mkisofs -quiet -r -V "SWC Sensor Install CD" \ - -cache-inodes \ - -J -l -b isolinux/isolinux.bin \ - -c isolinux/boot.cat -no-emul-boot \ - -boot-load-size 4 -boot-info-table \ - -eltorito-alt-boot -e boot/grub/efi.img -no-emul-boot \ - -o "../${ona_name}" local + $sudo cp ${ona_pkg_name} local/ona/${ona_pkg_name} + + echo "New format: $NEW_FORMAT " + if [ -n "$NEW_FORMAT" ]; then + # copy autoinstall folders for grub + $sudo cp -r ../autoinstall/nocloud-dhcp local/ + $sudo cp ../isolinux/grub.cfg local/boot/grub/grub.cfg + else + $sudo cp ../preseed/* local/preseed/ + $sudo cp ../isolinux/txt.cfg local/isolinux/txt.cfg + $sudo cp ../isolinux/grub.cfg local/boot/grub/grub.cfg + fi + + if [ -n "$NEW_FORMAT" ]; then + xorriso -as mkisofs -r -V 'SWC Sensor Install CD' \ + -o "../${ona_name}"\ + --grub2-mbr --interval:local_fs:0s-15s:zero_mbrpt,zero_gpt:"/root/${ubuntu_name}" \ + -partition_offset 16 \ + --mbr-force-bootable \ + -append_partition 2 0xef \ + --interval:local_fs:4099440d-4109507d::"/root/${ubuntu_name}" \ + -appended_part_as_gpt \ + -c "${BOOT_CAT}" \ + -b "${ELTORITO}" \ + -no-emul-boot -boot-load-size 4 -boot-info-table \ + --grub2-boot-info \ + -eltorito-alt-boot \ + -e '--interval:appended_partition_2:::' \ + -no-emul-boot \ + local + else + $sudo mkisofs -quiet -r -V "SWC Sensor Install CD" \ + -cache-inodes \ + -J -l -b "${BOOT_CAT}" \ + -c "${EFI}" -no-emul-boot \ + -joliet-long \ + -boot-load-size 4 -boot-info-table \ + -eltorito-alt-boot -e "${ELTORITO}" -no-emul-boot \ + -o "../${ona_name}" local + + isohybrid "../${ona_name}" + fi + $sudo umount cdrom $sudo chown $USER:$USER "../${ona_name}" - isohybrid "../${ona_name}" + $sudo rm -rf "$DIR"/working ) -$sudo rm -rf "$DIR"/working + diff --git a/images/iso/build_iso_helper b/images/iso/build_iso_helper index 465d826f..409d2942 100755 --- a/images/iso/build_iso_helper +++ b/images/iso/build_iso_helper @@ -17,6 +17,12 @@ $ ./build_iso_helper --doctests Real results as of February 2022: +>>> get_iso_url('24.04', subiquity=True) +https://releases.ubuntu.com/noble/ubuntu-24.04.1-live-server-amd64.iso + +>>> get_iso_url('22.04', subiquity=True) +'https://releases.ubuntu.com/jammy/ubuntu-22.04.4-live-server-amd64.iso' + >>> get_iso_url('20.04.3', subiquity=True) 'https://releases.ubuntu.com/20.04/ubuntu-20.04.3-live-server-amd64.iso' @@ -53,9 +59,10 @@ ValueError: no release found Traceback (most recent call last): ValueError: no Subiquity release for this version """ + from argparse import ArgumentParser, Action, SUPPRESS from doctest import DocTestSuite -from unittest import TestCase, TextTestRunner, makeSuite +from unittest import TestCase, TextTestRunner, TestLoader from unittest.mock import patch from urllib.request import Request, build_opener, HTTPRedirectHandler from urllib.error import URLError, HTTPError @@ -120,11 +127,14 @@ def get_iso_url(version, subiquity: bool, arch='amd64'): urlpath = f'https://releases.ubuntu.com/{v.short}/' isofile = f'ubuntu-{v.long_if_patch}-live-server-{arch}.iso' else: - urlpath = ( - 'https://cdimage.ubuntu.com/ubuntu-legacy-server/releases' - f'/{v.short}/release/' - ) - isofile = f'ubuntu-{v.long_if_patch}-legacy-server-{arch}.iso' + if v.major >= 22: + raise ValueError(f'No legacy ISO for version {v.short}') + else: + urlpath = ( + 'https://cdimage.ubuntu.com/ubuntu-legacy-server/releases' + f'/{v.short}/release/' + ) + isofile = f'ubuntu-{v.long_if_patch}-legacy-server-{arch}.iso' elif v.major >= 18: if subiquity: urlpath = f'https://releases.ubuntu.com/{v.short}/' @@ -181,7 +191,7 @@ class Tests(TestCase): def fake_head_status_code(url): if url in ( - f'{cd_legacy}/20.04.1/release/ubuntu-20.04.1-legacy-server-amd64.iso', + f'{cd_legacy}/20.04/release/ubuntu-20.04.1-legacy-server-amd64.iso', f'{releases}/20.04/ubuntu-20.04.1-live-server-amd64.iso', f'{old}/20.04.0/ubuntu-20.04-live-server-amd64.iso', 'https://www.google.com', @@ -195,9 +205,10 @@ class Tests(TestCase): get_iso_url('20.04.1', True), f'{releases}/20.04/ubuntu-20.04.1-live-server-amd64.iso', ) + self.assertEqual( get_iso_url('20.04.1', False), - f'{cd_legacy}/20.04.1/release/ubuntu-20.04.1-legacy-server-amd64.iso', + f'{cd_legacy}/20.04/release/ubuntu-20.04.1-legacy-server-amd64.iso', ) self.assertEqual( get_iso_url('20.04.0', True), @@ -210,7 +221,7 @@ class Tests(TestCase): class _TestAction(Action): def __init__(self, option_strings, dest, default=SUPPRESS, help=None): if dest == 'unittests': - self._suite = makeSuite(Tests) + self._suite = TestLoader().loadTestsFromTestCase(Tests) elif dest == 'doctests': self._suite = DocTestSuite() else: diff --git a/images/iso/isolinux/grub.cfg b/images/iso/isolinux/grub.cfg index f9fd0099..9ecf579c 100644 --- a/images/iso/isolinux/grub.cfg +++ b/images/iso/isolinux/grub.cfg @@ -1,23 +1,36 @@ - if loadfont /boot/grub/font.pf2 ; then - set gfxmode=auto - insmod efi_gop - insmod efi_uga - insmod gfxterm - terminal_output gfxterm + set gfxmode=auto + insmod efi_gop + insmod efi_uga + insmod gfxterm + terminal_output gfxterm fi set menu_color_normal=white/black set menu_color_highlight=black/light-gray set timeout=30 + +loadfont unicode + menuentry "Install ONA (Static IP)" { - set gfxpayload=keep - linux /install/vmlinuz file=/cdrom/preseed/nodhcp.seed quiet --- - initrd /install/initrd.gz -} -menuentry "Install ONA (DHCP)" { - set gfxpayload=keep - linux /install/vmlinuz file=/cdrom/preseed/dhcp.seed quiet --- - initrd /install/initrd.gz + set gfxpayload=keep + linux /casper/vmlinuz debug autoinstall ds=nocloud\;s=/cdrom/nocloud-dhcp/ --- + initrd /casper/initrd } + + + +grub_platform +if [ "$grub_platform" = "efi" ]; then + menuentry 'Boot from next volume' { + exit 1 + } + menuentry 'UEFI Firmware Settings' { + zo fwsetup + } +else + menuentry 'Test memory' { + linux16 /boot/memtest86+.bin + } +fi \ No newline at end of file diff --git a/images/iso/ona/configure.sh b/images/iso/ona/configure.sh index 32eefef3..3f09de19 100755 --- a/images/iso/ona/configure.sh +++ b/images/iso/ona/configure.sh @@ -27,4 +27,6 @@ mv /root/ona/motd.tail /etc/update-motd.d/01-obsrvbl # Install the ONA service package DEBIAN_FRONTEND=noninteractive apt install -y /root/ona/netsa-pkg.deb -DEBIAN_FRONTEND=noninteractive apt install -y /root/ona/ona-service.deb +DEBIAN_FRONTEND=noninteractive apt install -y /root/ona/ona-service_UbuntuNoble_amd64.deb + +/bin/sed -i '/^OBSRVBL_ONA_NAME=/ s/^/#/' /opt/obsrvbl-ona/config.local \ No newline at end of file diff --git a/images/packer/ona-service.json b/images/packer/ona-service.json index 45e842b8..e786bb50 100644 --- a/images/packer/ona-service.json +++ b/images/packer/ona-service.json @@ -19,7 +19,7 @@ "sleep 20", "sudo yum install --assumeyes squid tcpdump", "sudo chkconfig squid on", - "curl -O https://s3.amazonaws.com/onstatic/ona/master/ona-service_RHEL_6_amd64.rpm", + "curl -O https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.3/ona-service_RHEL_6_amd64.rpm", "sudo yum install --assumeyes ona-service_RHEL_6_amd64.rpm" ] }] diff --git a/package_builder.py b/package_builder.py index 285c570e..82bf3d96 100644 --- a/package_builder.py +++ b/package_builder.py @@ -30,9 +30,9 @@ SystemInfo('rpm', REDHAT_COMMON + ('net-tools', 'python3', 'sudo')), 'RaspbianJessie': SystemInfo('deb', RASBPI_COMMON), - 'UbuntuXenial': + 'UbuntuNoble': SystemInfo('deb', UBUNTU_COMMON + ('systemd-sysv', 'net-tools')), - 'UbuntuXenialContainer': + 'UbuntuNobleContainer': SystemInfo('deb', UBUNTU_COMMON + ('net-tools',)), } diff --git a/packaging/root/opt/obsrvbl-ona/system/obsrvbl_ona.sudoers b/packaging/root/opt/obsrvbl-ona/system/obsrvbl_ona.sudoers index b99af412..49419cde 100644 --- a/packaging/root/opt/obsrvbl-ona/system/obsrvbl_ona.sudoers +++ b/packaging/root/opt/obsrvbl-ona/system/obsrvbl_ona.sudoers @@ -5,7 +5,7 @@ Defaults:obsrvbl_ona !syslog obsrvbl_ona ALL=(ALL) NOPASSWD: /sbin/ifconfig obsrvbl_ona ALL=(ALL) NOPASSWD: /opt/obsrvbl-ona/pna/user/pna obsrvbl_ona ALL=(ALL) NOPASSWD: /opt/yaf/bin/yaf -obsrvbl_ona ALL=(ALL) NOPASSWD: /usr/sbin/tcpdump +obsrvbl_ona ALL=(ALL) NOPASSWD: /usr/bin/tcpdump obsrvbl_ona ALL=(suricata) NOPASSWD: /opt/suricata/manage.sh obsrvbl_ona ALL=(ALL) NOPASSWD: /sbin/ipset restore -exist -file /opt/obsrvbl-ona/system/netflow-udp.ipset obsrvbl_ona ALL=(ALL) NOPASSWD: /usr/sbin/ipset restore -exist -file /opt/obsrvbl-ona/system/netflow-udp.ipset diff --git a/packaging/root/opt/obsrvbl-ona/system/supervisord/ona-eta-monitor.sh b/packaging/root/opt/obsrvbl-ona/system/supervisord/ona-eta-monitor.sh index 31c017cb..f744c6f8 100755 --- a/packaging/root/opt/obsrvbl-ona/system/supervisord/ona-eta-monitor.sh +++ b/packaging/root/opt/obsrvbl-ona/system/supervisord/ona-eta-monitor.sh @@ -24,7 +24,7 @@ sleep `expr $OBSRVBL_ETA_CAPTURE_SECONDS - \`date +%s\` % $OBSRVBL_ETA_CAPTURE_S # Run the monitor exec /usr/bin/sudo \ - /usr/sbin/tcpdump \ + /usr/bin/tcpdump \ -w "$OBSRVBL_ETA_PCAP_DIR/logs_%s.pcap" \ -i "$OBSRVBL_ETA_CAPTURE_IFACE" \ -s 0 \ diff --git a/packaging/root/opt/obsrvbl-ona/system/supervisord/ona-pdns-monitor.sh b/packaging/root/opt/obsrvbl-ona/system/supervisord/ona-pdns-monitor.sh index 59cb67dd..a42f2a57 100755 --- a/packaging/root/opt/obsrvbl-ona/system/supervisord/ona-pdns-monitor.sh +++ b/packaging/root/opt/obsrvbl-ona/system/supervisord/ona-pdns-monitor.sh @@ -24,7 +24,7 @@ sleep `expr $OBSRVBL_PDNS_CAPTURE_SECONDS - \`date +%s\` % $OBSRVBL_PDNS_CAPTURE # Run the monitor exec /usr/bin/sudo \ - /usr/sbin/tcpdump \ + /usr/bin/tcpdump \ -w "$OBSRVBL_PDNS_PCAP_DIR/pdns_%s.pcap" \ -i "$OBSRVBL_PDNS_CAPTURE_IFACE" \ -s 0 \ diff --git a/packaging/scripts/postinst_UbuntuXenial.sh b/packaging/scripts/postinst_UbuntuXenial.sh deleted file mode 100644 index 68d152ce..00000000 --- a/packaging/scripts/postinst_UbuntuXenial.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/sh - -# Copyright 2015 Observable Networks -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -case "$1" in - abort-upgrade|abort-remove|abort-deconfigure|configure) - ;; - *) - echo "postinst called with unknown argument \`$1'" >&2 - exit 1 - ;; -esac - - -PYTHONPATH="/opt/obsrvbl-ona:/opt/obsrvbl-ona/system/python-packages" /usr/bin/python3 /opt/obsrvbl-ona/ona_service/installation/postinst.py "UbuntuXenial" diff --git a/packaging/scripts/postinst_UbuntuXenialContainer.sh b/packaging/scripts/postinst_UbuntuXenialContainer.sh deleted file mode 100644 index 5d0e066f..00000000 --- a/packaging/scripts/postinst_UbuntuXenialContainer.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/sh - -# Copyright 2015 Observable Networks -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -case "$1" in - abort-upgrade|abort-remove|abort-deconfigure|configure) - ;; - *) - echo "postinst called with unknown argument \`$1'" >&2 - exit 1 - ;; -esac - - -PYTHONPATH="/opt/obsrvbl-ona:/opt/obsrvbl-ona/system/python-packages" /usr/bin/python3 /opt/obsrvbl-ona/ona_service/installation/postinst.py "UbuntuXenialContainer" diff --git a/raspberry_pi_guide.md b/raspberry_pi_guide.md index b69d1753..2206ea19 100644 --- a/raspberry_pi_guide.md +++ b/raspberry_pi_guide.md @@ -17,7 +17,7 @@ revision: 0 1. Download the .deb for ONA and tar.gz's for SILK and YAF: ```bash cd /tmp -wget https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.2/ona-service_RaspbianJessie_armhf.deb +wget https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.3/ona-service_RaspbianJessie_armhf.deb wget https://tools.netsa.cert.org/releases/silk-3.19.2.tar.gz wget https://tools.netsa.cert.org/releases/yaf-2.12.1.tar.gz ``` diff --git a/src/scripts/ona_service/installation/postinst.py b/src/scripts/ona_service/installation/postinst.py index 2282c728..069aa041 100644 --- a/src/scripts/ona_service/installation/postinst.py +++ b/src/scripts/ona_service/installation/postinst.py @@ -47,7 +47,7 @@ def main(system_type): parser.add_argument( 'system_type', metavar='system_type', - help='Name of system class (e.g. UbuntuXenial)' + help='Name of system class (e.g. UbuntuNoble)' ) args = parser.parse_args() main(args.system_type) diff --git a/src/scripts/ona_service/installation/system_tools.py b/src/scripts/ona_service/installation/system_tools.py index 7a570953..5f0378b5 100644 --- a/src/scripts/ona_service/installation/system_tools.py +++ b/src/scripts/ona_service/installation/system_tools.py @@ -244,15 +244,15 @@ class RHEL_8(SystemdMixin, RedHatMixin, BaseSystem): """ -class UbuntuXenial(SystemdMixin, DebianMixin, BaseSystem): +class UbuntuNoble(SystemdMixin, DebianMixin, BaseSystem): """ Supports Ubuntu installations with the systemd init system. """ -class UbuntuXenialContainer(DebianMixin, BaseSystem): +class UbuntuNobleContainer(DebianMixin, BaseSystem): """ - Supports Ubuntu Xenial and above, but skips the systemd service steps. + Supports Ubuntu Noble and above, but skips the systemd service steps. For use with Docker, etc. """ def install_services(self): diff --git a/src/scripts/requirements.txt b/src/scripts/requirements.txt index 09435d2d..b9f0149a 100644 --- a/src/scripts/requirements.txt +++ b/src/scripts/requirements.txt @@ -1,9 +1,14 @@ -certifi==2023.7.22 -chardet==4.0.0 -idna==2.10 -python-dateutil==2.8.1 -requests==2.31.0 -retrying==1.3.3 -six==1.15.0 -supervisor==4.2.2 -urllib3==1.26.18 +pip>=24.3.1 +setuptools>=75.3.0 +wheel>=0.44.0 + +certifi==2024.8.30 +chardet==5.2.0 +idna==3.10 +pyasynchat==1.0.4 +python-dateutil==2.9.0 +requests==2.32.3 +retrying==1.3.4 +six==1.16.0 +supervisor==4.2.5 +urllib3==2.2.3 From 8d5591bb3dbabca2fb9909553214f33ae6a6da69 Mon Sep 17 00:00:00 2001 From: Dawid Kowalczyk Date: Thu, 21 Nov 2024 14:07:07 +0100 Subject: [PATCH 2/2] Update to 5.1.3 --- images/iso/autoinstall/README.md | 30 +++++++++ images/iso/autoinstall/nocloud-dhcp/meta-data | 0 images/iso/autoinstall/nocloud-dhcp/user-data | 65 +++++++++++++++++++ packaging/scripts/postinst_UbuntuNoble.sh | 29 +++++++++ .../scripts/postinst_UbuntuNobleContainer.sh | 29 +++++++++ 5 files changed, 153 insertions(+) create mode 100644 images/iso/autoinstall/README.md create mode 100644 images/iso/autoinstall/nocloud-dhcp/meta-data create mode 100644 images/iso/autoinstall/nocloud-dhcp/user-data create mode 100644 packaging/scripts/postinst_UbuntuNoble.sh create mode 100644 packaging/scripts/postinst_UbuntuNobleContainer.sh diff --git a/images/iso/autoinstall/README.md b/images/iso/autoinstall/README.md new file mode 100644 index 00000000..4aa2f983 --- /dev/null +++ b/images/iso/autoinstall/README.md @@ -0,0 +1,30 @@ +# Ubuntu Autoinstall + +Link: https://canonical-subiquity.readthedocs-hosted.com/en/latest/intro-to-autoinstall.html + +## DHCP/Static IP + +* Automated DHCP: `nocloud-dhcp/user-data` +* Static IP: `nocloud-nodhcp/user-data` (there is + *interactive-sections* which will invoke text UI to enter IP + address manually or select the DHCP). + + +## Note + +The `autoinstall.yaml` is expected to be present in the root of ISO +Image. Its format is missing main `autoinstall:` header in 22.04. In +later version (>= 24) it is expected to have different indentation: + +```yaml +# Autoinstall configuration +autoinstall: + version: 1 + +# Storage configuration with LVM + storage: + layout: + name: lvm +... +``` + diff --git a/images/iso/autoinstall/nocloud-dhcp/meta-data b/images/iso/autoinstall/nocloud-dhcp/meta-data new file mode 100644 index 00000000..e69de29b diff --git a/images/iso/autoinstall/nocloud-dhcp/user-data b/images/iso/autoinstall/nocloud-dhcp/user-data new file mode 100644 index 00000000..febc9a4e --- /dev/null +++ b/images/iso/autoinstall/nocloud-dhcp/user-data @@ -0,0 +1,65 @@ +#cloud-config +# https://ubuntu.com/server/docs/install/autoinstall-reference +autoinstall: + version: 1 + + interactive-sections: + - network + - identity + - locale + - keyboard + + + apt: + fallback: offline-install + preserve_sources_list: false + + + + + source: + search_drivers: false + id: ubuntu-server + + storage: + layout: + name: lvm + + + + + users: + - name: ubuntu + groups: [adm, cdrom, dip, plugdev, lxd, sudo] + lock-passwd: false + sudo: ALL=(ALL) NOPASSWD:ALL + shell: /bin/bash + locale: en_US.UTF-8 + timezone: UTC + + ssh: + install-server: true + allow-pw: true + ssh_pwauth: true + + resize_rootfs: false + + # iptables-persistent settings + early-commands: + - echo 'iptables-persistent iptables-persistent/autosave_v6 boolean false' | debconf-set-selections + - echo 'iptables-persistent iptables-persistent/autosave_v4 boolean false' | debconf-set-selections + + + + late-commands: + - rm -r /target/var/cache/apt + - cp -r /cdrom/apt /target/var/cache/ + - curtin in-target --target /target -- apt-get -yy install apt-transport-https iptables-persistent ipset libjansson4 libltdl7 liblzo2-2 libnet1 libyaml-0-2 nano ntp ntpdate snmp tcpdump net-tools libsnappy1v5 python3-dateutil + - | + if [ -d /sys/firmware/efi ]; then + apt-get install -y efibootmgr + efibootmgr -o $(efibootmgr | perl -n -e '/Boot(.+)\* ubuntu/ && print $1') + fi + - cp -r /cdrom/ona/ /target/root/ + - curtin in-target --target=/target -- bash -xv /root/ona/configure.sh + diff --git a/packaging/scripts/postinst_UbuntuNoble.sh b/packaging/scripts/postinst_UbuntuNoble.sh new file mode 100644 index 00000000..6d0de402 --- /dev/null +++ b/packaging/scripts/postinst_UbuntuNoble.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +# Copyright 2015 Observable Networks +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +case "$1" in + abort-upgrade|abort-remove|abort-deconfigure|configure) + ;; + *) + echo "postinst called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + + +PYTHONPATH="/opt/obsrvbl-ona:/opt/obsrvbl-ona/system/python-packages" /usr/bin/python3 /opt/obsrvbl-ona/ona_service/installation/postinst.py "UbuntuNoble" diff --git a/packaging/scripts/postinst_UbuntuNobleContainer.sh b/packaging/scripts/postinst_UbuntuNobleContainer.sh new file mode 100644 index 00000000..12d00470 --- /dev/null +++ b/packaging/scripts/postinst_UbuntuNobleContainer.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +# Copyright 2015 Observable Networks +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +case "$1" in + abort-upgrade|abort-remove|abort-deconfigure|configure) + ;; + *) + echo "postinst called with unknown argument \`$1'" >&2 + exit 1 + ;; +esac + + +PYTHONPATH="/opt/obsrvbl-ona:/opt/obsrvbl-ona/system/python-packages" /usr/bin/python3 /opt/obsrvbl-ona/ona_service/installation/postinst.py "UbuntuNobleContainer"