Datacomp submission improvement

components/resize_images/Dockerfile

@@ -0,0 +1,23 @@
+FROM --platform=linux/amd64 python:3.8-slim
+
+## System dependencies
+RUN apt-get update && \
+    apt-get upgrade -y && \
+    apt-get install git -y
+
+# install requirements
+COPY requirements.txt /
+RUN pip3 install --no-cache-dir -r requirements.txt
+
+# Install Fondant
+# This is split from other requirements to leverage caching
+ARG FONDANT_VERSION=main
+RUN pip3 install fondant[aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
+
+# Set the working directory to the component folder
+WORKDIR /component/src
+
+# Copy over src-files
+COPY src/ .
+
+ENTRYPOINT ["fondant", "execute", "main"]

components/resize_images/README.md

@@ -0,0 +1,45 @@
+# Resize images
+
+### Description
+Component that resizes images based on given width and height
+
+### Inputs / outputs
+
+**This component consumes:**
+
+- images
+    - data: binary
+
+**This component produces:**
+
+- images
+    - data: binary
+
+### Arguments
+
+The component takes the following arguments to alter its behavior:
+
+| argument | type | description | default |
+| -------- | ---- | ----------- | ------- |
+| resize_width | int | The width to resize to | / |
+| resize_height | int | The height to resize to | / |
+
+### Usage
+
+You can add this component to your pipeline using the following code:
+
+```python
+from fondant.pipeline import ComponentOp
+
+
+resize_images_op = ComponentOp.from_registry(
+    name="resize_images",
+    arguments={
+        # Add arguments
+        # "resize_width": 0,
+        # "resize_height": 0,
+    }
+)
+pipeline.add_op(resize_images_op, dependencies=[...])  #Add previous component as dependency
+```
+

components/resize_images/fondant_component.yaml

@@ -0,0 +1,23 @@
+name: Resize images
+description: Component that resizes images based on given width and height
+image: fndnt/resize_images:dev
+
+consumes:
+  images:
+    fields:
+      data:
+        type: binary
+
+produces:
+  images:
+    fields:
+      data:
+        type: binary
+
+args:
+  resize_width:
+    description: The width to resize to
+    type: int
+  resize_height:
+    description: The height to resize to
+    type: int

components/resize_images/requirements.txt

@@ -0,0 +1 @@
+Pillow==10.0.1

components/resize_images/src/main.py

@@ -0,0 +1,34 @@
+import io
+import logging
+
+import pandas as pd
+from fondant.component import PandasTransformComponent
+from PIL import Image
+
+logger = logging.getLogger(__name__)
+
+
+class ResizeImagesComponent(PandasTransformComponent):
+    """Component that resizes images based on a given width and height."""
+
+    def __init__(self, *_, resize_width: int, resize_height: int) -> None:
+        self.resize_width = resize_width
+        self.resize_height = resize_height
+
+    def resize_image(self, img: bytes):
+        image = Image.open(io.BytesIO(img)).convert("RGB")
+        image = image.resize((self.resize_width, self.resize_height))
+        image_bytes = io.BytesIO()
+        image.save(image_bytes, format="PNG")
+        return image_bytes.getvalue()
+
+    def transform(self, dataframe: pd.DataFrame) -> pd.DataFrame:
+        logger.info("Resizing images...")
+        result = dataframe["images"].apply(
+            lambda x: self.resize_image(x.data),
+            axis=1,
+        )
+
+        dataframe[("images", "data")] = result
+
+        return dataframe

examples/pipelines/datacomp/components/add_clip_score/Dockerfile

@@ -11,7 +11,7 @@ RUN pip3 install --no-cache-dir -r requirements.txt
 
 # Install Fondant
 # This is split from other requirements to leverage caching
-ARG FONDANT_VERSION=main
+ARG FONDANT_VERSION=0.6.2
 RUN pip3 install fondant[aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
 
 # Set the working directory to the component folder

examples/pipelines/datacomp/components/add_clip_score/fondant_component.yaml

@@ -1,6 +1,6 @@
 name: Add CLIP score
 description: Component that adds the CLIP score
-image: fndnt/add_clip_score:dev
+image: fndnt/add_clip_score:0.6.2
 
 consumes:
   embeddings:

examples/pipelines/datacomp/components/detect_text/Dockerfile

@@ -15,7 +15,7 @@ RUN pip3 install --no-cache-dir -r requirements.txt
 
 # Install Fondant
 # This is split from other requirements to leverage caching
-ARG FONDANT_VERSION=main
+ARG FONDANT_VERSION=0.6.2
 RUN pip3 install fondant[aws,azure,gcp]@git+https://github.com/ml6team/fondant@${FONDANT_VERSION}
 
 # Set the working directory to the component folder

examples/pipelines/datacomp/components/detect_text/fondant_component.yaml

@@ -1,6 +1,6 @@
 name: Detect text
 description: Component that detects text in images using an mmocr model
-image: fndnt/detect_text:dev
+image: fndnt/detect_text:0.6.2
 
 consumes:
   images:

examples/pipelines/datacomp/components/detect_text/src/config/fast/nas-configs/fast_tiny.config

@@ -0,0 +1,231 @@
+{
+    "name": "ProxylessNASNets",
+    "bn": {
+        "momentum": 0.1,
+        "eps": 0.001
+    },
+    "first_conv": {
+        "name": "ConvLayer",
+        "kernel_size": 3,
+        "stride": 2,
+        "dilation": 1,
+        "groups": 1,
+        "bias": false,
+        "has_shuffle": false,
+        "in_channels": 3,
+        "out_channels": 64,
+        "use_bn": true,
+        "act_func": "relu",
+        "dropout_rate": 0,
+        "ops_order": "weight_bn_act"
+    },
+    "stage1": [
+        {
+            "name": "RepConvLayer",
+            "in_channels": 64,
+            "out_channels": 64,
+            "kernel_size": [3, 3],
+            "stride": 1,
+            "dilation": 1,
+            "groups": 1
+        },
+        {
+            "name": "RepConvLayer",
+            "in_channels": 64,
+            "out_channels": 64,
+            "kernel_size": [3, 3],
+            "stride": 2,
+            "dilation": 1,
+            "groups": 1
+        },
+        {
+            "name": "RepConvLayer",
+            "in_channels": 64,
+            "out_channels": 64,
+            "kernel_size": [3, 3],
+            "stride": 1,
+            "dilation": 1,
+            "groups": 1
+        }
+    ],
+    "stage2": [
+        {
+            "name": "RepConvLayer",
+            "in_channels": 64,
+            "out_channels": 128,
+            "kernel_size": [3, 3],
+            "stride": 2,
+            "dilation": 1,
+            "groups": 1
+        },
+        {
+            "name": "RepConvLayer",
+            "in_channels": 128,
+            "out_channels": 128,
+            "kernel_size": [1, 3],
+            "stride": 1,
+            "dilation": 1,
+            "groups": 1
+        },
+        {
+            "name": "RepConvLayer",
+            "in_channels": 128,
+            "out_channels": 128,
+            "kernel_size": [3, 3],
+            "stride": 1,
+            "dilation": 1,
+            "groups": 1
+        },
+        {
+            "name": "RepConvLayer",
+            "in_channels": 128,
+            "out_channels": 128,
+            "kernel_size": [3, 1],
+            "stride": 1,
+            "dilation": 1,
+            "groups": 1
+        }
+    ],
+    "stage3": [
+        {
+            "name": "RepConvLayer",
+            "in_channels": 128,
+            "out_channels": 256,
+            "kernel_size": [3, 3],
+            "stride": 2,
+            "dilation": 1,
+            "groups": 1
+        },
+        {
+            "name": "RepConvLayer",
+            "in_channels": 256,
+            "out_channels": 256,
+            "kernel_size": [3, 3],
+            "stride": 1,
+            "dilation": 1,
+            "groups": 1
+        },
+        {
+            "name": "RepConvLayer",
+            "in_channels": 256,
+            "out_channels": 256,
+            "kernel_size": [3, 1],
+            "stride": 1,
+            "dilation": 1,
+            "groups": 1
+        },
+        {
+            "name": "RepConvLayer",
+            "in_channels": 256,
+            "out_channels": 256,
+            "kernel_size": [1, 3],
+            "stride": 1,
+            "dilation": 1,
+            "groups": 1
+        }
+    ],
+    "stage4": [
+        {
+            "name": "RepConvLayer",
+            "in_channels": 256,
+            "out_channels": 512,
+            "kernel_size": [3, 3],
+            "stride": 2,
+            "dilation": 1,
+            "groups": 1
+        },
+        {
+            "name": "RepConvLayer",
+            "in_channels": 512,
+            "out_channels": 512,
+            "kernel_size": [3, 1],
+            "stride": 1,
+            "dilation": 1,
+            "groups": 1
+        },
+        {
+            "name": "RepConvLayer",
+            "in_channels": 512,
+            "out_channels": 512,
+            "kernel_size": [1, 3],
+            "stride": 1,
+            "dilation": 1,
+            "groups": 1
+        },
+        {
+            "name": "RepConvLayer",
+            "in_channels": 512,
+            "out_channels": 512,
+            "kernel_size": [3, 3],
+            "stride": 1,
+            "dilation": 1,
+            "groups": 1
+        }
+    ],
+    "head": {
+        "name": "Head",
+        "conv": {
+            "name": "RepConvLayer",
+            "in_channels": 512,
+            "out_channels": 128,
+            "kernel_size": [3, 3],
+            "stride": 1,
+            "dilation": 1,
+            "groups": 1
+        },
+        "final": {
+            "name": "ConvLayer",
+            "kernel_size": 1,
+            "stride": 1,
+            "dilation": 1,
+            "groups": 1,
+            "bias": false,
+            "has_shuffle": false,
+            "in_channels": 128,
+            "out_channels": 5,
+            "use_bn": false,
+            "act_func": null,
+            "dropout_rate": 0,
+            "ops_order": "weight"
+        }
+    },
+    "neck": {
+        "name": "Neck",
+        "reduce_layer1": {
+            "name": "RepConvLayer",
+            "in_channels": 64,
+            "out_channels": 128,
+            "kernel_size": [3, 3],
+            "stride": 1,
+            "dilation": 1,
+            "groups": 1
+        },
+        "reduce_layer2": {
+            "name": "RepConvLayer",
+            "in_channels": 128,
+            "out_channels": 128,
+            "kernel_size": [3, 3],
+            "stride": 1,
+            "dilation": 1,
+            "groups": 1
+        },
+        "reduce_layer3": {
+            "name": "RepConvLayer",
+            "in_channels": 256,
+            "out_channels": 128,
+            "kernel_size": [3, 3],
+            "stride": 1,
+            "dilation": 1,
+            "groups": 1
+        },
+        "reduce_layer4": {
+            "name": "RepConvLayer",
+            "in_channels": 512,
+            "out_channels": 128,
+            "kernel_size": [3, 3],
+            "stride": 1,
+            "dilation": 1,
+            "groups": 1
+        }
+    }
+}