first commit

.ipynb_checkpoints/Dockerfile-checkpoint

@@ -111,6 +111,7 @@ RUN useradd -m -u 1000 user
 RUN mkdir -p /home/user/execution_engine
 COPY --chown=user ./execution_engine /home/user/execution_engine
 COPY --chown=user $WORKER_CFG_DB /home/user
+RUN chmod -R 777 /home/user/execution_engine
 ENV WORKER_CFG_DB /home/user/worker_cfg_db.csv
 
 USER user

Dockerfile

@@ -111,6 +111,7 @@ RUN useradd -m -u 1000 user
 RUN mkdir -p /home/user/execution_engine
 COPY --chown=user ./execution_engine /home/user/execution_engine
 COPY --chown=user $WORKER_CFG_DB /home/user
+RUN chmod -R 777 /home/user/execution_engine
 ENV WORKER_CFG_DB /home/user/worker_cfg_db.csv
 
 USER user

Dockerfile.bak

@@ -1,119 +0,0 @@
-FROM ubuntu:jammy
-WORKDIR /root
-
-ENV DEBIAN_FRONTEND noninteractive 
-ENV TZ Etc/UTC
-
-RUN apt-get update && \
-    apt-get install -y \
-        gnupg ca-certificates apt-utils \ 
-        build-essential curl unzip && \
-    apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF 
-RUN echo "deb https://download.mono-project.com/repo/ubuntu stable-focal main" | tee /etc/apt/sources.list.d/mono-official-stable.list 
-RUN apt-get update && \
-    apt-get install -y tzdata
-
-RUN apt-get install -y mono-devel \
-        gcc-12 g++-12
-
-RUN apt-get update && \
-    apt-get install -y software-properties-common python3.11-dev 
-
-RUN apt-get update && \
-    apt-get install -y -f libasound2 libc6-i386 libc6-x32 libxi6 libxtst6
-
-ENV JAVA_PKG=https://download.oracle.com/java/21/latest/jdk-21_linux-x64_bin.tar.gz \
-  JAVA_HOME=/usr/java/jdk-21
-
-RUN set -eux; \
-  JAVA_SHA256=$(curl "$JAVA_PKG".sha256) ; \
-  curl --output /tmp/jdk.tgz "$JAVA_PKG" && \
-  echo "$JAVA_SHA256 */tmp/jdk.tgz" | sha256sum -c; \
-  mkdir -p "$JAVA_HOME"; \
-  tar --extract --file /tmp/jdk.tgz --directory "$JAVA_HOME" --strip-components 1
-
-RUN curl -OL https://github.com/JetBrains/kotlin/releases/download/v1.7.20/kotlin-compiler-1.7.20.zip
-RUN unzip kotlin-compiler-1.7.20.zip -d /usr/local && \
-    rm kotlin-compiler-1.7.20.zip
-
-RUN apt-get install -y python2
-
-RUN curl -OL https://go.dev/dl/go1.19.2.linux-amd64.tar.gz
-RUN tar -C /usr/local -xzf go1.19.2.linux-amd64.tar.gz && \
-    rm go1.19.2.linux-amd64.tar.gz
-
-RUN curl -OL https://downloads.python.org/pypy/pypy3.9-v7.3.9-linux64.tar.bz2
-RUN tar -C /usr/local -xf pypy3.9-v7.3.9-linux64.tar.bz2 && \
-    rm pypy3.9-v7.3.9-linux64.tar.bz2
-
-RUN curl -OL https://downloads.python.org/pypy/pypy2.7-v7.3.9-linux64.tar.bz2
-RUN tar -C /usr/local -xf pypy2.7-v7.3.9-linux64.tar.bz2 && \
-    rm pypy2.7-v7.3.9-linux64.tar.bz2
-
-RUN apt-get install -y clang-14
-
-ARG NODE_VERSION=v16.17.1
-ARG NODE_DISTRO=linux-x64
-ARG NODE_ZIP=node-$NODE_VERSION-$NODE_DISTRO.tar.xz
-RUN curl -OL https://nodejs.org/dist/$NODE_VERSION/$NODE_ZIP
-RUN tar -C /usr/local -xJvf $NODE_ZIP && \
-    rm $NODE_ZIP
-
-ENV RUSTUP_HOME /opt/rust
-ENV CARGO_HOME /opt/rust
-RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --no-modify-path
-RUN apt-get install -y ruby-full
-
-RUN apt-get update && apt-get install -y --no-install-recommends --fix-missing php8.1
-RUN apt-get install -y libgmp-dev libmpfr-dev 
-
-ENV PATH $PATH:/usr/local/go/bin:/usr/local/kotlinc/bin:/usr/local/node-v16.17.1-linux-x64/bin:/opt/rust/bin
-
-WORKDIR /root
-
-
-RUN update-alternatives --install /usr/bin/java java "$JAVA_HOME"/bin/java 100 && \
-    update-alternatives --install /usr/bin/javac javac "$JAVA_HOME"/bin/javac 100 && \
-    update-alternatives --install /usr/bin/jar jar "$JAVA_HOME"/bin/jar 100 && \
-    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 100 && \
-    update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 100 && \
-    update-alternatives --install /usr/bin/python python /usr/bin/python3.11 100 && \
-    update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 100 && \
-    update-alternatives --install /usr/bin/pypy2 pypy2 /usr/local/pypy2.7-v7.3.9-linux64/bin/pypy2 100 && \
-    update-alternatives --install /usr/bin/pypy3 pypy3 /usr/local/pypy3.9-v7.3.9-linux64/bin/pypy3 100 
-
-RUN apt-get install -y python3-pip gperf
-RUN pip install flask gunicorn flask-cors gmpy2 Cython jsonlines fire 
-
-RUN curl -o libseccomp.tar.gz -L https://github.com/seccomp/libseccomp/releases/download/v2.5.4/libseccomp-2.5.4.tar.gz
-RUN tar -xzvf libseccomp.tar.gz && cd libseccomp-2.5.4 && chmod +x configure
-WORKDIR /root/libseccomp-2.5.4
-RUN ./configure --prefix=/usr --enable-python && make
-RUN make install
-WORKDIR /root/
-RUN rm -rf libseccomp*
-
-
-ENV RUN_UID 1586
-ENV RUN_GID 1586
-ENV NUM_WORKERS 2
-ENV GUNICORN_PORT 7860
-ENV LOG_LEVEL info
-
-# RUN groupadd -g ${RUN_GID} runner${RUN_GID} && useradd -M runner${RUN_UID} -g ${RUN_GID} -u ${RUN_UID}
-
-EXPOSE ${GUNICORN_PORT}
-RUN useradd -m -u 1000 user
-
-RUN mkdir /home/user/execution_engine
-COPY --chown=user ./execution_engine /home/user/execution_engine
-COPY --chown=user $WORKER_CFG_DB /home/user
-ENV WORKER_CFG_DB /home/user/worker_cfg_db.csv
-RUN mkdir -p /home/user/code_store
-RUN chown -R user:user /code_store
-
-USER user
-ENV PATH="/home/user/.local/bin:$PATH"
-WORKDIR /home/user/execution_engine
-
-CMD ["bash", "start_engine.sh"]

execution_engine/.ipynb_checkpoints/code_store-checkpoint.py

@@ -0,0 +1,54 @@
+import os
+import shutil
+import uuid
+from pathlib import Path
+
+from config import CodeStoreConfig
+
+
+class CodeStore:
+    _source_dir: Path
+
+    def __init__(self, cfg: CodeStoreConfig, run_ids: tuple[int, int]) -> None:
+        self._source_dir = cfg.source_code_dir / uuid.uuid4().hex
+        self.uid = run_ids[1]
+        self.gid = run_ids[0]
+
+    def create(self):
+        # os.makedirs(self._source_dir, exist_ok=True)
+        # os.chown(self._source_dir, self.uid, self.gid)
+        # os.chmod(self._source_dir, 0o775)
+        return None
+
+    def destroy(self) -> None:
+        shutil.rmtree(self._source_dir, ignore_errors=True)
+
+    def write_source_code(self, source_code: str, filename: Path) -> Path:
+        filepath = self._source_dir / filename
+
+        with filepath.open("w") as fp:
+            fp.write(source_code)
+
+        filepath = filepath.resolve()
+
+        os.chown(filepath, self.uid, self.gid)
+        os.chmod(filepath, 0o775)
+        return filepath
+
+    def read_source_code(self, filepath: Path) -> str:
+        with filepath.open() as f:
+            s = f.read()
+
+        return s
+
+
+if __name__ == "__main__":
+    from config import load_config
+
+    cfg = load_config(Path("execution_engine/config.yaml"))
+    code_store = CodeStore(cfg.code_store)
+    print(
+        code_store.read_source_code(
+            code_store.write_source_code("""print("Hello")""", Path("main.py"))
+        )
+    )

execution_engine/code_store.py

@@ -15,9 +15,10 @@ class CodeStore:
         self.gid = run_ids[0]
 
     def create(self):
-        os.makedirs(self._source_dir, exist_ok=True)
-        os.chown(self._source_dir, self.uid, self.gid)
-        os.chmod(self._source_dir, 0o775)
+        # os.makedirs(self._source_dir, exist_ok=True)
+        # os.chown(self._source_dir, self.uid, self.gid)
+        # os.chmod(self._source_dir, 0o775)
+        return None
 
     def destroy(self) -> None:
         shutil.rmtree(self._source_dir, ignore_errors=True)