Merge pull request #38 from obsrvbl-oss/update_public_repository

Update to v5.1.3

.gitignore

@@ -21,6 +21,7 @@ index.html
 .coverage
 
 images/iso/*.iso
+images/iso/local_files/
 packaging/output/
 packaging/root/opt/obsrvbl-ona/netflow/
 packaging/root/opt/obsrvbl-ona/ipfix/

Makefile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 ARCH ?= amd64
-VERSION := 5.1.2
+VERSION := 5.1.3
 
 SCRIPTS_DIR := src/scripts
 uPNA_DIR := src/uPNA
@@ -71,13 +71,13 @@ ona-service_RaspbianJessie_%.deb:
 	mkdir -p $(dir $@)
 	python package_builder.py $(notdir $*) ${VERSION} RaspbianJessie
 
-ona-service_UbuntuXenial_%.deb:
+ona-service_UbuntuNoble_%.deb:
 	mkdir -p $(dir $@)
-	python package_builder.py $(notdir $*) ${VERSION} UbuntuXenial
+	python package_builder.py $(notdir $*) ${VERSION} UbuntuNoble
 
-ona-service_UbuntuXenialContainer_%.deb:
+ona-service_UbuntuNobleContainer_%.deb:
 	mkdir -p $(dir $@)
-	python package_builder.py $(notdir $*) ${VERSION} UbuntuXenialContainer
+	python package_builder.py $(notdir $*) ${VERSION} UbuntuNobleContainer
 
 .PHONY: clean
 clean:

README.md

@@ -2,24 +2,28 @@
 
 This repository is where the development of the Observable Networks Appliance (ONA) takes place. The ONA software is used to collect input data for Observable Networks' network security service. It can run on a variety of platforms, including embedded computers, physical servers, virtual machines, cloud servers, and Docker containers.
 
-## Supported platforms
+## Download
 
-The following platforms are officially supported:
+### ISO (fully supported and recommended):
 
-* [Ubuntu 18.04 and later](https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.2/ona-service_UbuntuXenial_amd64.deb)
-* [RHEL 7 and compatible](https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.2/ona-service_RHEL_7_x86_64.rpm)
-* [RHEL 8 and compatible](https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.2/ona-service_RHEL_8_x86_64.rpm)
-* [Raspberry Pi with Raspbian (ARMHF)](https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.2/ona-service_RaspbianJessie_armhf.deb)
+* [Ubuntu 24.04](https://assets-production.obsrvbl.com/ona-packages/iso/ona-24.04.1-v5.1.3/ona-24.04.1-server-amd64.iso)
+
+### Package files for manual installation:
+
+* [Ubuntu 24.04 and later](https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.3/ona-service_UbuntuNoble_amd64.deb)
+* [RHEL 7 and compatible](https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.3/ona-service_RHEL_7_x86_64.rpm)
+* [RHEL 8 and compatible](https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.3/ona-service_RHEL_8_x86_64.rpm)
+* [Raspberry Pi with Raspbian (ARMHF)](https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.3/ona-service_RaspbianJessie_armhf.deb)
   ([installation guide](raspberry_pi_guide.md))
-* [Raspberry Pi with Raspbian (ARM64)](https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.2/ona-service_RaspbianJessie_aarch64.deb)
+* [Raspberry Pi with Raspbian (ARM64)](https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.3/ona-service_RaspbianJessie_aarch64.deb)
   ([installation guide](raspberry_pi_guide.md))
 * [Docker](https://github.com/obsrvbl/ona/blob/master/images/docker/Dockerfile)
 
-To install the latest version on 20.04 (recommended for physical and virtual machine installations):
+To install the latest version on Ubuntu:
 
 ```
-$ wget https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.2/ona-service_UbuntuXenial_amd64.deb
-$ sudo apt install ./ona-service_UbuntuXenial_amd64.deb
+$ wget https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.3/ona-service_UbuntuNoble_amd64.deb
+$ sudo apt install ./ona-service_UbuntuNoble_amd64.deb
 ```
 
 To monitor NetFlow traffic, you'll also need to install tools from the [CERT NetSA Security Suite](https://tools.netsa.cert.org/):

images/docker/Dockerfile

@@ -48,15 +48,15 @@ RUN curl -L -O https://assets-production.obsrvbl.com/ona-packages/netsa/v0.1.27/
     && rm -rf netsa-pkg.deb
 
 # Use local copy of ONA service package if needed
-# COPY ona-service_UbuntuXenialContainer_amd64.deb ./
+# COPY ona-service_UbuntuNobleContainer_amd64.deb ./
 
 # Install ONA service
-RUN if [ ! -f ./ona-service_UbuntuXenialContainer_amd64.deb ] ;\
-    then curl -L -O https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.2/ona-service_UbuntuXenialContainer_amd64.deb ;\
+RUN if [ ! -f ./ona-service_UbuntuNobleContainer_amd64.deb ] ;\
+    then curl -L -O https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.3/ona-service_UbuntuNobleContainer_amd64.deb ;\
     else echo "Use cached package" ;fi \
-    && apt-get update && apt-get install --assume-yes --fix-missing ./ona-service_UbuntuXenialContainer_amd64.deb \
+    && apt-get update && apt-get install --assume-yes --fix-missing ./ona-service_UbuntuNobleContainer_amd64.deb \
     && rm -rf /var/lib/apt/lists/* \
-    && rm -rf ona-service_UbuntuXenialContainer_amd64.deb
+    && rm -rf ona-service_UbuntuNobleContainer_amd64.deb
 
 # Switch to the unprivileged user, set some local configuration, and start.
 COPY run.sh /opt/obsrvbl-ona/run.sh

images/iso/autoinstall/README.md

@@ -0,0 +1,30 @@
+# Ubuntu Autoinstall
+
+Link: https://canonical-subiquity.readthedocs-hosted.com/en/latest/intro-to-autoinstall.html
+
+## DHCP/Static IP
+
+* Automated DHCP: `nocloud-dhcp/user-data`
+* Static IP: `nocloud-nodhcp/user-data` (there is
+  *interactive-sections* which will invoke text UI to enter IP
+  address manually or select the DHCP).
+
+
+## Note
+
+The `autoinstall.yaml` is expected to be present in the root of ISO
+Image. Its format is missing main `autoinstall:` header in 22.04. In
+later version (>= 24) it is expected to have different indentation:
+
+```yaml
+# Autoinstall configuration
+autoinstall:
+  version: 1
+
+# Storage configuration with LVM
+  storage:
+    layout:
+      name: lvm
+...
+```
+

images/iso/autoinstall/nocloud-dhcp/user-data

@@ -0,0 +1,65 @@
+#cloud-config
+# https://ubuntu.com/server/docs/install/autoinstall-reference
+autoinstall:
+  version: 1
+
+  interactive-sections:
+   - network
+   - identity
+   - locale
+   - keyboard
+
+
+  apt:
+    fallback: offline-install
+    preserve_sources_list: false
+
+
+
+
+  source:
+      search_drivers: false
+      id: ubuntu-server
+
+  storage:
+    layout:
+      name: lvm
+
+
+
+
+    users:
+      - name: ubuntu
+        groups: [adm, cdrom, dip, plugdev, lxd, sudo]
+        lock-passwd: false
+        sudo: ALL=(ALL) NOPASSWD:ALL
+        shell: /bin/bash
+  locale: en_US.UTF-8
+  timezone: UTC
+
+  ssh:
+    install-server: true
+    allow-pw: true
+    ssh_pwauth: true
+
+  resize_rootfs: false
+
+  # iptables-persistent settings
+  early-commands:
+    - echo 'iptables-persistent iptables-persistent/autosave_v6 boolean false' | debconf-set-selections
+    - echo 'iptables-persistent iptables-persistent/autosave_v4 boolean false' | debconf-set-selections
+
+
+
+  late-commands:
+    - rm -r /target/var/cache/apt
+    - cp -r /cdrom/apt /target/var/cache/
+    - curtin in-target --target /target -- apt-get -yy install apt-transport-https iptables-persistent ipset libjansson4 libltdl7 liblzo2-2 libnet1 libyaml-0-2 nano ntp ntpdate snmp tcpdump net-tools libsnappy1v5 python3-dateutil
+    - |
+      if [ -d /sys/firmware/efi ]; then
+        apt-get install -y efibootmgr
+        efibootmgr -o $(efibootmgr | perl -n -e '/Boot(.+)\* ubuntu/ && print $1')
+      fi
+    - cp -r /cdrom/ona/ /target/root/
+    - curtin in-target --target=/target -- bash -xv /root/ona/configure.sh
+

images/iso/build_iso.sh

@@ -21,9 +21,9 @@
 #    wrong.
 #
 
-RELEASE="${RELEASE:-20.04.1}"
+RELEASE="${RELEASE:-24.04.1}"
 ARCH="${ARCH:-amd64}"
-VARIANT="${VARIANT:-legacy}"
+VARIANT="${VARIANT:-subiquity}"
 
 
 DIR=$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)
@@ -45,47 +45,138 @@ while getopts "f:a:r:" opt ; do
            ;;
     esac
 done
+# Newly added
+ ubuntu_name="ubuntu-${RELEASE}-live-server-${ARCH}.iso"
+ ona_name="ona-${RELEASE}-server-${ARCH}.iso"
+ ubuntu_url="${url:-$($DIR/build_iso_helper $RELEASE $VARIANT)}"
+
+# ubuntu_name="ubuntu-24.04.1-live-server-amd64.iso"
+# ona_name="ona-${RELEASE}-server-${ARCH}.iso"
+ONA_URL="https://s3.amazonaws.com/onstatic/ona-service/master/"
+if [ -n "$PUBLIC_ONA" ]; then
+  ONA_URL="https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.2/"
+fi
+# netsa_pkg_name="netsa-pkg.deb"
+ona_pkg_name="ona-service_UbuntuNoble_amd64.deb"
 
-ubuntu_name="ubuntu-${RELEASE}-server-${ARCH}.iso"
-ona_name="ona-${RELEASE}-server-${ARCH}.iso"
-ubuntu_url="${url:-$($DIR/build_iso_helper $RELEASE $VARIANT)}"
 test -n "$ubuntu_url" || fatal "failed getting Ubuntu ISO download URL"
-ona_service_url="https://s3.amazonaws.com/onstatic/ona-service/master/ona-service_UbuntuXenial_amd64.deb"
-netsa_pkg_url="https://assets-production.obsrvbl.com/ona-packages/netsa/v0.1.27/netsa-pkg.deb"
+
+ ONA_URL="https://s3.amazonaws.com/onstatic/ona-service/master/"
+ if [ -n "$PUBLIC_ONA" ]; then
+   ONA_URL="https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.2/"
+
+ fi
+
+ #ona_service_url="${ONA_URL}ona-service_UbuntuNoble_amd64.deb"
+ ona_service_url="https://assets-production.obsrvbl.com/ona-packages/obsrvbl-ona/v5.1.3/ona-service_UbuntuNoble_amd64.deb"
+ netsa_pkg_url="https://assets-production.obsrvbl.com/ona-packages/netsa/v0.1.27/netsa-pkg.deb"
+
+
 
 shift $(($OPTIND-1))
 
 test $EUID -ne 0 && sudo="sudo"
-which mkisofs 1> /dev/null || fatal "missing mkisofs: $sudo apt-get install genisoimage"
-which isohybrid 1> /dev/null || fatal "missing isohybrid: $sudo apt-get install syslinux-utils"
 
 [[ -d "$DIR" ]] || fatal  # invalid directory
-[[ -d "$DIR"/working && $(ls -A "$DIR"/working) ]] && fatal  # working directory exists and is not empty
 [[ -d "$DIR"/working ]] || mkdir "$DIR"/working # working directory does not exist, so create it
+
+major_version=$(echo "$RELEASE" | cut -d '.' -f 1)
+
+# Check if the major version number is greater than 20
+if [ "$major_version" -gt 20 ]; then
+  which xorriso 1> /dev/null || fatal "missing xorriso: $sudo apt-get install xorriso -y"
+  NEW_FORMAT=true
+  BOOT_CAT="/boot.catalog"
+  EFI='/boot/grub/i386-pc/eltorito.img'
+  ELTORITO='/boot/grub/i386-pc/eltorito.img'
+else
+  which mkisofs 1> /dev/null || fatal "missing mkisofs: $sudo apt-get install genisoimage"
+  which isohybrid 1> /dev/null || fatal "missing isohybrid: $sudo apt-get install syslinux-utils"
+  BOOT_CAT="isolinux/boot.cat"
+  EFI="isolinux/isolinux.bin"
+  ELTORITO="boot/grub/efi.img"
+fi
+
 (
   set -e
+  if [ ! -e "/root/$ubuntu_name" ]; then
+    curl -L -o /root/${ubuntu_name} "${ubuntu_url}"
+  fi
+
   cd "$DIR"/working
-  curl -L -o ${ubuntu_name} "${ubuntu_url}"
+  #[[ -d "$DIR/local_files/" ]] && cp "$DIR"/local_files/* .
   curl -L -o netsa-pkg.deb "${netsa_pkg_url}"
-  curl -L -o ona-service.deb "${ona_service_url}"
+  #curl -L -o "${ona_pkg_name}" "${ona_service_url}"
+  $sudo cp /obsrvbl/images/iso/ona-service_UbuntuNoble_amd64.deb /obsrvbl/images/iso/working/
+
+
+
+
+
+$sudo apt-get -y update
+# you can install packages here if you want
+
+PACKAGES="apt-transport-https iptables-persistent ipset libjansson4 libltdl7 liblzo2-2 libnet1 libyaml-0-2 nano ntp ntpdate snmp tcpdump net-tools libsnappy1v5 python3-dateutil"
+$sudo apt-get -yyqq install --download-only ${PACKAGES}
+
+
+
+
+  # local is root dir in ISO
   mkdir cdrom local
-  $sudo mount -o loop --read-only "${ubuntu_name}" cdrom
+  pwd
+
+  $sudo mount -o loop --read-only "/root/${ubuntu_name}" cdrom
   rsync -av --quiet cdrom/ local
-  $sudo cp ../preseed/* local/preseed/
+
+  $sudo cp -r /var/cache/apt local
   $sudo cp -r ../ona local
   $sudo cp netsa-pkg.deb local/ona/netsa-pkg.deb
-  $sudo cp ona-service.deb local/ona/ona-service.deb
-  $sudo cp ../isolinux/txt.cfg local/isolinux/txt.cfg
-  $sudo cp ../isolinux/grub.cfg local/boot/grub/grub.cfg
-  $sudo mkisofs -quiet -r -V "SWC Sensor Install CD" \
-          -cache-inodes \
-          -J -l -b isolinux/isolinux.bin \
-          -c isolinux/boot.cat -no-emul-boot \
-          -boot-load-size 4 -boot-info-table \
-          -eltorito-alt-boot -e boot/grub/efi.img -no-emul-boot \
-          -o "../${ona_name}" local
+  $sudo cp ${ona_pkg_name} local/ona/${ona_pkg_name}
+
+  echo "New format: $NEW_FORMAT "
+  if [ -n "$NEW_FORMAT" ]; then
+    # copy autoinstall folders for grub
+    $sudo cp -r ../autoinstall/nocloud-dhcp  local/
+    $sudo cp ../isolinux/grub.cfg local/boot/grub/grub.cfg
+  else
+    $sudo cp ../preseed/* local/preseed/
+    $sudo cp ../isolinux/txt.cfg local/isolinux/txt.cfg
+    $sudo cp ../isolinux/grub.cfg local/boot/grub/grub.cfg
+  fi
+
+  if [ -n "$NEW_FORMAT" ]; then
+    xorriso -as mkisofs -r  -V 'SWC Sensor Install CD' \
+      -o "../${ona_name}"\
+      --grub2-mbr --interval:local_fs:0s-15s:zero_mbrpt,zero_gpt:"/root/${ubuntu_name}" \
+      -partition_offset 16 \
+      --mbr-force-bootable \
+      -append_partition 2 0xef \
+      --interval:local_fs:4099440d-4109507d::"/root/${ubuntu_name}" \
+      -appended_part_as_gpt \
+      -c "${BOOT_CAT}" \
+      -b "${ELTORITO}" \
+      -no-emul-boot -boot-load-size 4 -boot-info-table \
+      --grub2-boot-info \
+      -eltorito-alt-boot \
+      -e '--interval:appended_partition_2:::' \
+      -no-emul-boot \
+      local
+  else
+    $sudo mkisofs -quiet -r -V "SWC Sensor Install CD" \
+      -cache-inodes \
+      -J -l -b "${BOOT_CAT}" \
+      -c "${EFI}" -no-emul-boot \
+      -joliet-long \
+      -boot-load-size 4 -boot-info-table \
+      -eltorito-alt-boot -e "${ELTORITO}" -no-emul-boot \
+      -o "../${ona_name}" local
+
+    isohybrid "../${ona_name}"
+  fi
+
   $sudo umount cdrom
   $sudo chown $USER:$USER "../${ona_name}"
-  isohybrid "../${ona_name}"
+  $sudo rm -rf "$DIR"/working
 )
-$sudo rm -rf "$DIR"/working
+