From e182eebcc59ab690b33372726801ac66c9866c57 Mon Sep 17 00:00:00 2001 From: bluepal-yaswanth-peravali Date: Thu, 18 Dec 2025 13:30:31 +0530 Subject: [PATCH] added-nodejs-express-support --- CHANGELOG.md | 53 +- Dockerfile.express.template | 23 + baseimages/Dockerfile.node22base | 45 +- baseimages/scripts/entrypoint.sh | 4 + docs/ARCHITECTURE_EXPRESS.md | 240 ++++++++ docs/SERVICEMAKER_OVERVIEW_AND_RISKS.md | 453 +++++++++++++++ docs/SERVICE_COMPARISON.md | 527 ++++++++++++++++++ scripts/prepareproject-express.sh | 79 +++ src/main.rs | 264 ++++++++- .../shoppinglist-express/.env.example | 4 + testprojects/shoppinglist-express/README.md | 53 ++ testprojects/shoppinglist-express/index.js | 226 ++++++++ .../shoppinglist-express/package.json | 19 + testprojects/shoppinglist-express/schemas.js | 10 + testprojects/shoppinglist-express/swagger.js | 52 ++ 15 files changed, 2010 insertions(+), 42 deletions(-) create mode 100644 Dockerfile.express.template create mode 100644 docs/ARCHITECTURE_EXPRESS.md create mode 100644 docs/SERVICEMAKER_OVERVIEW_AND_RISKS.md create mode 100644 docs/SERVICE_COMPARISON.md create mode 100644 scripts/prepareproject-express.sh create mode 100644 testprojects/shoppinglist-express/.env.example create mode 100644 testprojects/shoppinglist-express/README.md create mode 100644 testprojects/shoppinglist-express/index.js create mode 100644 testprojects/shoppinglist-express/package.json create mode 100644 testprojects/shoppinglist-express/schemas.js create mode 100644 testprojects/shoppinglist-express/swagger.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 23f49a0..632e230 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,32 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +#### Express Application Support +- **Express Project Detection**: Added automatic detection of Express.js applications + - Detects Express apps by checking for `express` dependency in `package.json` + - Requires absence of `services.json` and `manifest.json` (distinguishes from Foxx services) + - Project type: `express` + +- **Express Dockerfile Template**: Created `Dockerfile.express.template` for Express applications + - Uses Node.js 22 base image (`arangodb/node22base:latest`) + - Runs Express apps directly with `node {ENTRYPOINT}` instead of `node-foxx` + - No `services.json` or `manifest.json` required + - Entrypoint auto-detected from `package.json` `main` field or `start` script + - Defaults to `index.js` if not found + +- **Express Preparation Script**: Added `scripts/prepareproject-express.sh` + - Similar to Node.js preparation but without `node-foxx` checks + - Installs only missing/incompatible dependencies + - Uses base `node_modules` from base image via NODE_PATH + +- **Environment Variables from `.env.example`**: Added support for reading environment variables + - Automatically reads `.env.example` file if present in project root + - Parses `KEY=VALUE` format with support for quoted values + - Injects environment variables as `ENV` directives in Dockerfile + - Supports comments (lines starting with `#`) and empty lines + - Handles values with spaces or special characters (auto-quotes for Docker) + - Works for all project types (Python, Foxx, Express) + #### Node.js/Foxx Service Support - **Node.js Base Image**: Added `Dockerfile.node22base` for Node.js 22 base image with pre-installed packages - Installs Node.js 22 from NodeSource @@ -36,9 +62,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Project Type Detection**: Extended `detect_project_type()` to support: - `python`: Projects with `pyproject.toml` + - `express`: Express.js applications with `express` dependency, no `services.json` or `manifest.json` - `foxx`: Multi-service projects with `package.json` and `services.json` (both required) - `foxx-service`: Single service directory with `package.json` only (auto-generates `services.json`) - - Execution stops with error if `services.json` is missing for Node.js projects - **Service Structure Generation**: Simplified structure for single service directories - Copies service directory directly to `/project/{service-name}/` (no wrapper folder) @@ -66,16 +92,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed -- **Main Application Logic**: Extended `src/main.rs` to support Node.js/Foxx projects +- **Main Application Logic**: Extended `src/main.rs` to support Node.js/Foxx and Express projects + - Added Express project type detection and handling + - Added entrypoint auto-detection for Express apps from `package.json` + - Added environment variable reading from `.env.example` for all project types + - Added Dockerfile modification functions for Express apps (`modify_dockerfile_express`) + - Added Express preparation script to embedded scripts list - Added project type detection requiring both `package.json` and `services.json` for `foxx` type - - Error handling: execution stops if `services.json` is missing for Node.js projects - Simplified file copying: projects are copied as-is (no wrapper structure generation) - Base image default handling: Introduced compile-time constants (`DEFAULT_PYTHON_BASE_IMAGE`, `DEFAULT_NODEJS_BASE_IMAGE`) - Explicit user intent tracking: Changed `base_image` to `Option` to detect explicit user choices - Smart defaults: Only sets project-type-specific defaults when user hasn't explicitly set base image - - Modified Dockerfile generation to use Node.js template for Foxx projects - - Updated Helm chart generation to support Node.js/Foxx projects - - Added `prepareproject-nodejs.sh` and `check-base-dependencies.js` to embedded scripts list + - Modified Dockerfile generation to use appropriate template based on project type + - Updated Helm chart generation to support all project types (Python, Express, Foxx) + - Added `prepareproject-express.sh` to embedded scripts list - No entrypoint required for Foxx services (uses `node-foxx` from base image) - **Entrypoint Script**: Enhanced `baseimages/scripts/entrypoint.sh` to support Node.js/Foxx services @@ -96,7 +126,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Technical Details -For detailed information about base image structure, service architecture, and module resolution, see [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md). +For detailed information about: +- **Node.js/Foxx Services**: Base image structure, service architecture, and module resolution - see [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) +- **Express Applications**: Express app architecture, detection, and deployment - see [docs/ARCHITECTURE_EXPRESS.md](docs/ARCHITECTURE_EXPRESS.md) +- **Service Comparison**: Differences between Node-Foxx and Express+Arangojs services - see [docs/SERVICE_COMPARISON.md](docs/SERVICE_COMPARISON.md) ## [0.9.2] - Previous Release @@ -113,14 +146,16 @@ For detailed information about base image structure, service architecture, and m ## Version History - **0.9.2**: Initial release with Python support -- **Unreleased**: Added Node.js/Foxx service support +- **Unreleased**: Added Node.js/Foxx service support and Express application support --- ## Notes - All changes maintain backward compatibility with existing Python projects -- Node.js support is additive and does not affect Python service functionality +- Node.js support (both Foxx and Express) is additive and does not affect Python service functionality +- Express applications are detected automatically and require no special configuration files +- Environment variables from `.env.example` are automatically injected into Docker images - Base images must be built separately using `baseimages/build.sh` - Windows users should use WSL or Linux environment for building base images diff --git a/Dockerfile.express.template b/Dockerfile.express.template new file mode 100644 index 0000000..863b56f --- /dev/null +++ b/Dockerfile.express.template @@ -0,0 +1,23 @@ +FROM {BASE_IMAGE} + +USER root + +COPY ./scripts /scripts +COPY {PROJECT_DIR} /project/{PROJECT_DIR} +RUN chown -R user:user /project/{PROJECT_DIR} + +USER user +WORKDIR /project/{WORKDIR} + +# Set NODE_PATH to resolve from project node_modules first, then base node_modules +# This allows npm to install only missing/incompatible packages in project directory +# while still accessing base packages from /home/user/node_modules +ENV NODE_PATH={NODE_PATH} + +RUN /scripts/prepareproject-express.sh + +EXPOSE {PORT} + +# Run Express app directly using node +CMD ["node", "{ENTRYPOINT}"] + diff --git a/baseimages/Dockerfile.node22base b/baseimages/Dockerfile.node22base index e023852..7646565 100644 --- a/baseimages/Dockerfile.node22base +++ b/baseimages/Dockerfile.node22base @@ -16,28 +16,29 @@ USER user # Standard packages are selected to benefit most projects while keeping image size reasonable RUN npm init -y && \ npm install \ - # ArangoDB/Foxx core - @arangodb/node-foxx@^0.0.1-alpha.0 \ - @arangodb/node-foxx-launcher@^0.0.1-alpha.0 \ - @arangodb/arangodb@^0.0.1-alpha.0 \ - # Dependency checking utility - semver@^7.6.3 \ - # Essential utilities - lodash@^4.17.21 \ - dayjs@^1.11.10 \ - uuid@^9.0.1 \ - dotenv@^16.4.5 \ - # HTTP clients - axios@^1.7.2 \ - # Validation - joi@^17.13.3 \ - # Logging - winston@^3.15.0 \ - # Async utilities - async@^3.2.5 \ - # Security - jsonwebtoken@^9.0.2 \ - bcrypt@^5.1.1 + # ArangoDB/Foxx core + @arangodb/node-foxx@^0.0.1-alpha.0 \ + @arangodb/node-foxx-launcher@^0.0.1-alpha.0 \ + @arangodb/arangodb@^0.0.1-alpha.0 \ + arangojs@^10.1.2 \ + # Dependency checking utility + semver@^7.6.3 \ + # Essential utilities + lodash@^4.17.21 \ + dayjs@^1.11.10 \ + uuid@^9.0.1 \ + dotenv@^16.4.5 \ + # HTTP clients + axios@^1.7.2 \ + # Validation + joi@^17.13.3 \ + # Logging + winston@^3.15.0 \ + # Async utilities + async@^3.2.5 \ + # Security + jsonwebtoken@^9.0.2 \ + bcrypt@^5.1.1 # Create checksums for base node_modules (for tracking, base remains immutable) RUN find node_modules -type f -print0 | \ diff --git a/baseimages/scripts/entrypoint.sh b/baseimages/scripts/entrypoint.sh index 0c38dd2..71fd138 100755 --- a/baseimages/scripts/entrypoint.sh +++ b/baseimages/scripts/entrypoint.sh @@ -32,6 +32,10 @@ if test -e entrypoint ; then echo "Error: node-foxx not found. Make sure node_modules are installed." exit 1 fi + elif [ -f "package.json" ] && [ ! -f "services.json" ] && [ ! -f "manifest.json" ] && grep -q '"express"' package.json 2>/dev/null; then + # Express.js application + echo "Detected Express.js application" + exec node $ENTRYPOINT else # Python service (existing logic) echo "Detected Python service" diff --git a/docs/ARCHITECTURE_EXPRESS.md b/docs/ARCHITECTURE_EXPRESS.md new file mode 100644 index 0000000..46411be --- /dev/null +++ b/docs/ARCHITECTURE_EXPRESS.md @@ -0,0 +1,240 @@ +# Express Application Architecture + +This document describes the technical architecture and design decisions for Express.js applications in ServiceMaker. + +## Overview + +Express applications are standalone Node.js web services that use Express.js framework and Arangojs driver directly, without the Foxx framework. They are simpler and more flexible than Foxx services, requiring no `services.json` or `manifest.json` files. + +## Project Detection + +ServiceMaker automatically detects Express applications by checking: + +1. **Presence of `package.json`**: Required for all Node.js projects +2. **Express dependency**: Checks if `express` is listed in `dependencies` or `devDependencies` +3. **Absence of Foxx files**: No `services.json` or `manifest.json` present + +**Detection Logic:** +```rust +if package_json.exists() { + if !services_json.exists() && !manifest_json.exists() { + if has_express_dependency(package_json) { + return "express" + } + } +} +``` + +## Project Structure + +Express applications have a simple, standard Node.js structure: + +``` +project-root/ +├── package.json # Dependencies and metadata +├── index.js # Main entry point (or custom) +├── .env.example # Optional: Environment variable template +├── routes/ # Optional: Route handlers +├── middleware/ # Optional: Express middleware +├── config/ # Optional: Configuration files +└── ... # Other application files +``` + +**Key Differences from Foxx Services:** +- No `services.json` required +- No `manifest.json` required +- No `node-foxx` or `node-foxx-launcher` dependencies needed +- Standard Express.js application structure + +## Base Image + +Express applications use the same Node.js base image as Foxx services: + +- **Base Image**: `arangodb/node22base:latest` +- **Node.js Version**: 22.x +- **Pre-installed Packages**: Standard Node.js packages (lodash, axios, joi, etc.) +- **Module Resolution**: Uses NODE_PATH for efficient dependency management + +## Entrypoint Detection + +ServiceMaker automatically detects the entrypoint for Express applications: + +**Priority Order:** +1. `package.json` `main` field (e.g., `"main": "index.js"`) +2. `package.json` `scripts.start` field (extracts from `"node index.js"` → `"index.js"`) +3. Default: `"index.js"` + +**Example:** +```json +{ + "name": "my-express-app", + "main": "server.js", + "scripts": { + "start": "node server.js" + } +} +``` +→ Entrypoint: `server.js` + +## Dockerfile Structure + +Express applications use `Dockerfile.express.template`: + +```dockerfile +FROM arangodb/node22base:latest + +USER root +COPY ./scripts /scripts +COPY {PROJECT_DIR} /project/{PROJECT_DIR} +RUN chown -R user:user /project/{PROJECT_DIR} + +USER user +WORKDIR /project/{WORKDIR} + +ENV NODE_PATH=/project/{PROJECT_DIR}/node_modules:/home/user/node_modules + +RUN /scripts/prepareproject-express.sh + +EXPOSE {PORT} + +CMD ["node", "{ENTRYPOINT}"] +``` + +**Key Features:** +- Runs directly with `node` (not `node-foxx`) +- Uses same dependency management as Foxx services +- Environment variables from `.env.example` are injected as `ENV` directives + +## Dependency Management + +Express applications use the same efficient dependency management as Foxx services: + +**Process:** +1. **Pre-install Analysis**: `check-base-dependencies.js` analyzes project dependencies +2. **Version Compatibility**: Checks if base packages satisfy project requirements +3. **Selective Installation**: Only installs missing or incompatible packages +4. **Module Resolution**: Uses NODE_PATH to resolve from both locations + +**Benefits:** +- Smaller `node_modules` (only project-specific packages) +- Faster builds (fewer packages to install) +- Base packages pre-scanned for security + +## Environment Variables + +Express applications support environment variables from `.env.example`: + +**File Format:** +``` +# Database configuration +ARANGO_DB_ENDPOINT=http://127.0.0.1:8529 +ARANGO_DB_NAME=_system + +# Application settings +PORT=3000 +NODE_ENV=production + +# Optional: Values with spaces +APP_NAME="My Express App" +``` + +**Processing:** +- Comments (lines starting with `#`) are ignored +- Empty lines are skipped +- Quoted values (single or double quotes) are unquoted +- Values with spaces or special characters are auto-quoted for Docker + +**Dockerfile Injection:** +```dockerfile +ENV NODE_PATH=/project/{PROJECT_DIR}/node_modules:/home/user/node_modules +ENV ARANGO_DB_ENDPOINT=http://127.0.0.1:8529 +ENV ARANGO_DB_NAME=_system +ENV PORT=3000 +ENV NODE_ENV=production +ENV APP_NAME="My Express App" +``` + +## Runtime Execution + +**Container Startup:** +1. **Entrypoint**: `node {ENTRYPOINT}` (e.g., `node index.js`) +2. **Working Directory**: `/project/{project-dir}/` +3. **Module Resolution**: Uses NODE_PATH to resolve dependencies +4. **Environment Variables**: Available from Docker ENV directives + +**Example:** +```bash +# Container runs: +cd /project/my-express-app +node index.js +``` + +## Comparison with Foxx Services + +| Feature | Express Apps | Foxx Services | +|---------|-------------|---------------| +| **Framework** | Express.js | Foxx (node-foxx) | +| **Orchestration** | None (standalone) | node-foxx-launcher | +| **Configuration** | Standard Express | manifest.json + services.json | +| **Entrypoint** | `node {file}` | `node-foxx` | +| **Multi-service** | No | Yes (via services.json) | +| **Worker Threads** | No | Yes (per service) | +| **Routing** | Express router | Foxx router | +| **API Docs** | Manual (Swagger) | Auto-generated (Swagger) | +| **Dependencies** | Standard npm | Foxx + npm | + +## Use Cases + +**Choose Express when:** +- You want standard Express.js patterns +- You don't need Foxx-specific features (manifest, services.json) +- You prefer simpler, more flexible architecture +- You want full control over application structure +- You're migrating from existing Express applications + +**Choose Foxx when:** +- You need multi-service orchestration +- You want automatic API documentation +- You need Foxx-specific features (context, dependencies) +- You're building ArangoDB-native services +- You want worker thread isolation + +## Best Practices + +1. **Project Structure**: Follow Express.js best practices + - Separate routes, middleware, and controllers + - Use environment variables for configuration + - Implement proper error handling + +2. **Dependencies**: Minimize project-specific packages + - Use base image packages when possible + - Check base packages before adding new dependencies + +3. **Environment Variables**: Use `.env.example` for documentation + - Document all required environment variables + - Provide default values where appropriate + - Keep sensitive values out of version control + +4. **Entrypoint**: Use `package.json` `main` field + - Makes entrypoint explicit and discoverable + - Works with standard Node.js tooling + +5. **Error Handling**: Implement comprehensive error handling + - Use Express error handling middleware + - Return consistent error responses + - Log errors appropriately + +## Migration from Foxx + +If you have an existing Foxx service and want to migrate to Express: + +1. **Remove Foxx Dependencies**: Remove `@arangodb/node-foxx` and related packages +2. **Replace Router**: Convert Foxx router to Express router +3. **Update Middleware**: Convert Foxx middleware to Express middleware +4. **Remove Configuration**: Remove `manifest.json` and `services.json` +5. **Update Database Access**: Use Arangojs directly instead of Foxx context +6. **Add Express**: Add `express` and `arangojs` to dependencies +7. **Update Entrypoint**: Change from `node-foxx` to `node index.js` + +See [SERVICE_COMPARISON.md](SERVICE_COMPARISON.md) for detailed differences. + diff --git a/docs/SERVICEMAKER_OVERVIEW_AND_RISKS.md b/docs/SERVICEMAKER_OVERVIEW_AND_RISKS.md new file mode 100644 index 0000000..ec8e635 --- /dev/null +++ b/docs/SERVICEMAKER_OVERVIEW_AND_RISKS.md @@ -0,0 +1,453 @@ +# ServiceMaker Overview and Security Risks + +## What is ServiceMaker? + +ServiceMaker is a Rust-based tool that automates the packaging and deployment preparation of microservices for containerized environments. It transforms source code projects into production-ready Docker images and Kubernetes deployment artifacts. + +### Core Functionality + +ServiceMaker takes a project directory and: + +1. **Detects Project Type**: Automatically identifies the project type: + - **Python**: Projects with `pyproject.toml` + - **Express.js**: Node.js applications with `express` dependency (no Foxx configuration) + - **Foxx Services**: Node.js services using `@arangodb/node-foxx` framework + - **Foxx-Service**: Single Foxx service (auto-generates `services.json`) + +2. **Reads Project Metadata**: Extracts information from: + - `package.json` (Node.js projects) + - `pyproject.toml` (Python projects) + - `.env.example` (environment variables) + +3. **Generates Docker Images**: + - Uses pre-built base images with common dependencies + - Copies project source code + - Installs only missing/incompatible dependencies (efficient layer caching) + - Injects environment variables from `.env.example` + - Sets appropriate entrypoints and working directories + +4. **Generates Helm Charts**: Creates Kubernetes deployment manifests including: + - Deployment configurations + - Service definitions + - Route/Ingress configurations + - Resource limits and requests + +5. **Optional Artifacts**: + - Docker image push to registry + - `project.tar.gz` archive creation + +### Supported Project Types + +| Type | Detection | Base Image | Entrypoint | +|------|-----------|------------|------------| +| **Python** | `pyproject.toml` | `arangodb/py13base:latest` | Python script | +| **Express** | `express` in dependencies, no `services.json` | `arangodb/node22base:latest` | `node {entrypoint}` | +| **Foxx** | `package.json` + `services.json` | `arangodb/node22base:latest` | `node-foxx` | +| **Foxx-Service** | `package.json` only | `arangodb/node22base:latest` | `node-foxx` | + +### Base Image Strategy + +ServiceMaker uses immutable base images that: +- Pre-install common dependencies (lodash, axios, joi, etc.) +- Are pre-scanned for security vulnerabilities +- Provide efficient layer caching +- Reduce build times and image sizes + +Projects only install packages that are: +- Missing from the base image +- Have incompatible versions with base packages + +## Deployment Pipeline + +``` +User Project + ↓ +ServiceMaker + ├──→ Docker Image (built) + ├──→ Helm Chart (generated) + └──→ Optional: tar.gz archive + ↓ +Docker Registry + ↓ +Kubernetes Cluster (via Helm) + ↓ +Running Service +``` + +## Security and Operational Risks + +When accepting user-provided services and deploying them on your organization's platform, several critical risks must be addressed: + +### 1. Code Execution Risks + +#### **Malicious Code Injection** +- **Risk**: User code can execute arbitrary commands, access filesystem, make network requests +- **Impact**: + - Data exfiltration + - System compromise + - Lateral movement in the cluster + - Cryptocurrency mining + - Botnet participation + +#### **Supply Chain Attacks** +- **Risk**: Malicious dependencies in `package.json` or `pyproject.toml` +- **Impact**: + - Compromised dependencies can execute during installation + - Backdoors in third-party packages + - Dependency confusion attacks + +**Mitigation Strategies:** +- ✅ **Code Review**: Mandatory security review of all user code before acceptance +- ✅ **Dependency Scanning**: Automated scanning of all dependencies (npm audit, pip-audit, Snyk, etc.) +- ✅ **SBOM Generation**: Software Bill of Materials for all dependencies +- ✅ **Whitelist Dependencies**: Restrict allowed packages to approved lists +- ✅ **Sandboxed Builds**: Build Docker images in isolated environments +- ✅ **Runtime Security**: Use Pod Security Policies, SecurityContext, and read-only filesystems + +### 2. Container Security Risks + +#### **Privilege Escalation** +- **Risk**: Containers running with excessive privileges +- **Impact**: Container escape, host system compromise + +**Mitigation:** +- ✅ Run containers as non-root user (ServiceMaker base images use `user` user) +- ✅ Set `securityContext.runAsNonRoot: true` in Helm charts +- ✅ Disable privilege escalation: `securityContext.allowPrivilegeEscalation: false` +- ✅ Drop all capabilities: `securityContext.capabilities.drop: ["ALL"]` + +#### **Resource Exhaustion** +- **Risk**: Malicious or buggy code consuming excessive resources +- **Impact**: + - Denial of Service (DoS) to other services + - Cluster resource exhaustion + - Cost overruns + +**Mitigation:** +- ✅ **Resource Limits**: Set CPU and memory limits in Helm charts +- ✅ **Quotas**: Implement namespace-level resource quotas +- ✅ **Monitoring**: Real-time resource usage monitoring and alerts +- ✅ **Auto-scaling Limits**: Configure maximum replica limits + +#### **Image Vulnerabilities** +- **Risk**: Vulnerable base images or dependencies +- **Impact**: Known CVEs exploited in production + +**Mitigation:** +- ✅ **Base Image Scanning**: Regular security scans of base images +- ✅ **Dependency Scanning**: Scan all installed packages +- ✅ **Regular Updates**: Keep base images updated with security patches +- ✅ **Vulnerability Database**: Integrate with CVE databases (NVD, GitHub Advisory) + +### 3. Network Security Risks + +#### **Unauthorized Network Access** +- **Risk**: Services making unauthorized outbound connections +- **Impact**: + - Data exfiltration + - Command and control (C2) communication + - Lateral movement + +**Mitigation:** +- ✅ **Network Policies**: Kubernetes NetworkPolicies to restrict traffic + - Default deny all egress + - Whitelist only required destinations + - Restrict inter-pod communication +- ✅ **Service Mesh**: Use Istio/Linkerd for fine-grained traffic control +- ✅ **Egress Filtering**: Firewall rules and proxy policies +- ✅ **DNS Policies**: Restrict DNS resolution to internal services + +#### **Inbound Attack Surface** +- **Risk**: Exposed services vulnerable to external attacks +- **Impact**: + - API abuse + - DDoS attacks + - Injection attacks + +**Mitigation:** +- ✅ **Ingress Controls**: Use Ingress controllers with rate limiting +- ✅ **WAF**: Web Application Firewall for HTTP/HTTPS traffic +- ✅ **Authentication**: Require authentication for all endpoints +- ✅ **Rate Limiting**: Implement per-user/IP rate limits +- ✅ **Input Validation**: Enforce strict input validation + +### 4. Data Security Risks + +#### **Sensitive Data Exposure** +- **Risk**: Services accessing or exposing sensitive data +- **Impact**: + - PII leakage + - Credential exposure + - Database access violations + +**Mitigation:** +- ✅ **Secrets Management**: Use Kubernetes Secrets or external secret managers (Vault, AWS Secrets Manager) +- ✅ **RBAC**: Role-Based Access Control for database and API access +- ✅ **Data Encryption**: Encrypt data at rest and in transit +- ✅ **Audit Logging**: Log all data access attempts +- ✅ **Data Loss Prevention (DLP)**: Monitor for sensitive data patterns + +#### **Database Access Abuse** +- **Risk**: Services with excessive database permissions +- **Impact**: + - Unauthorized data access + - Data modification/deletion + - Database performance degradation + +**Mitigation:** +- ✅ **Least Privilege**: Grant minimum required database permissions +- ✅ **Connection Pooling**: Limit concurrent database connections +- ✅ **Query Monitoring**: Monitor and alert on suspicious queries +- ✅ **Database Firewall**: Restrict database access by IP/service + +### 5. Configuration Security Risks + +#### **Environment Variable Injection** +- **Risk**: Malicious environment variables from `.env.example` +- **Impact**: + - Configuration manipulation + - Credential injection + - Path traversal attacks + +**Mitigation:** +- ✅ **Environment Variable Validation**: Review all environment variables +- ✅ **Secrets Separation**: Never allow secrets in `.env.example` +- ✅ **Configuration Review**: Manual review of all configuration +- ✅ **Immutable Config**: Use ConfigMaps and Secrets, not environment variables + +#### **Helm Chart Manipulation** +- **Risk**: User-provided Helm values could override security settings +- **Impact**: + - Disabled security policies + - Resource limit removal + - Privilege escalation + +**Mitigation:** +- ✅ **Helm Chart Validation**: Validate generated Helm charts +- ✅ **Value Constraints**: Restrict allowed Helm values +- ✅ **Template Security**: Review Helm chart templates for security +- ✅ **Policy Enforcement**: Use OPA Gatekeeper or Kyverno for policy enforcement + +### 6. Operational Risks + +#### **Service Availability** +- **Risk**: Buggy or unstable services causing outages +- **Impact**: + - Service downtime + - Cascading failures + - User impact + +**Mitigation:** +- ✅ **Health Checks**: Implement proper liveness and readiness probes +- ✅ **Circuit Breakers**: Implement circuit breakers for external dependencies +- ✅ **Graceful Shutdown**: Handle SIGTERM properly +- ✅ **Monitoring**: Comprehensive monitoring and alerting +- ✅ **Canary Deployments**: Gradual rollout with automatic rollback + +#### **Logging and Observability** +- **Risk**: Insufficient logging or log injection attacks +- **Impact**: + - Difficult incident response + - Log poisoning + - Compliance violations + +**Mitigation:** +- ✅ **Structured Logging**: Enforce structured logging standards +- ✅ **Log Aggregation**: Centralized logging (ELK, Loki, etc.) +- ✅ **Log Retention**: Appropriate log retention policies +- ✅ **Sensitive Data Filtering**: Filter PII and secrets from logs +- ✅ **Audit Trails**: Maintain audit trails for compliance + +#### **Update and Patching** +- **Risk**: Outdated dependencies or base images +- **Impact**: + - Known vulnerabilities in production + - Compliance violations + +**Mitigation:** +- ✅ **Automated Scanning**: Continuous vulnerability scanning +- ✅ **Patch Management**: Automated patch deployment process +- ✅ **Version Pinning**: Pin dependency versions for reproducibility +- ✅ **Update Policies**: Define update and patching policies + +### 7. Compliance and Legal Risks + +#### **License Violations** +- **Risk**: Services using incompatible licenses +- **Impact**: Legal liability, license compliance issues + +**Mitigation:** +- ✅ **License Scanning**: Automated license scanning (FOSSA, Snyk) +- ✅ **License Policies**: Define allowed license types +- ✅ **Attribution**: Maintain license attribution files + +#### **Data Privacy Violations** +- **Risk**: Services violating GDPR, CCPA, or other regulations +- **Impact**: Regulatory fines, legal liability + +**Mitigation:** +- ✅ **Data Classification**: Classify and tag data appropriately +- ✅ **Privacy Impact Assessments**: Conduct PIAs for new services +- ✅ **Data Residency**: Ensure data residency compliance +- ✅ **Right to Deletion**: Implement data deletion capabilities + +## Recommended Security Controls + +### Pre-Deployment + +1. **Mandatory Code Review** + - Security team review of all code + - Automated static analysis (SAST) + - Dependency scanning + - License compliance checking + +2. **Build-Time Security** + - Sandboxed build environments + - Base image scanning + - Dependency vulnerability scanning + - SBOM generation + +3. **Image Security** + - Image signing and verification + - Image scanning (Trivy, Clair, etc.) + - Minimal base images + - Multi-stage builds + +### Deployment-Time + +1. **Kubernetes Security** + - Pod Security Standards (restricted profile) + - Network Policies (default deny) + - RBAC (least privilege) + - Resource quotas and limits + +2. **Service Mesh** + - mTLS between services + - Traffic policies + - Observability + +3. **Secrets Management** + - External secret managers + - Secret rotation + - Encrypted at rest + +### Runtime + +1. **Monitoring and Alerting** + - Resource usage monitoring + - Security event detection + - Anomaly detection + - Real-time alerts + +2. **Runtime Security** + - Runtime threat detection (Falco, Aqua) + - File integrity monitoring + - Network traffic analysis + - Behavioral analysis + +3. **Incident Response** + - Automated incident response playbooks + - Forensic capabilities + - Log retention for investigation + +## ServiceMaker Security Enhancements + +### Current Security Features + +✅ **Non-root User**: Base images run as non-root `user` +✅ **Immutable Base Images**: Base `node_modules` is read-only +✅ **Dependency Analysis**: Only installs missing/incompatible packages +✅ **Environment Variable Parsing**: Validates `.env.example` format + +### Recommended Enhancements + +1. **Dependency Scanning Integration** + - Integrate npm audit, pip-audit into build process + - Fail builds on high/critical vulnerabilities + - Generate vulnerability reports + +2. **Image Signing** + - Sign Docker images with cosign + - Verify signatures before deployment + - Integrate with registry policies + +3. **SBOM Generation** + - Generate SPDX or CycloneDX SBOMs + - Attach SBOMs to images + - Store SBOMs in artifact registry + +4. **Security Context Injection** + - Automatically inject security contexts in Helm charts + - Enforce non-root, read-only filesystems + - Drop all capabilities by default + +5. **Network Policy Generation** + - Generate default-deny NetworkPolicies + - Require explicit allow rules + - Document network requirements + +6. **Resource Limit Enforcement** + - Set default resource limits + - Require resource requests + - Prevent resource exhaustion + +7. **Secrets Validation** + - Detect secrets in code (truffleHog, git-secrets) + - Validate secret references + - Prevent secret hardcoding + +## Best Practices for Accepting User Services + +### 1. Establish Clear Policies + +- **Acceptance Criteria**: Define what services are acceptable +- **Security Requirements**: Minimum security standards +- **Resource Limits**: Default resource constraints +- **Compliance Requirements**: Regulatory requirements + +### 2. Implement Security Gates + +- **Automated Scanning**: All code and dependencies scanned +- **Manual Review**: Security team approval required +- **Testing Requirements**: Unit tests, integration tests +- **Documentation Requirements**: API documentation, runbooks + +### 3. Continuous Monitoring + +- **Runtime Monitoring**: Resource usage, errors, latency +- **Security Monitoring**: Unusual network activity, file access +- **Compliance Monitoring**: Data access, audit logs +- **Cost Monitoring**: Resource consumption, cost attribution + +### 4. Incident Response + +- **Playbooks**: Documented response procedures +- **Isolation**: Ability to quickly isolate compromised services +- **Forensics**: Logging and evidence collection +- **Communication**: Stakeholder notification procedures + +### 5. Regular Audits + +- **Security Audits**: Regular security assessments +- **Compliance Audits**: Regulatory compliance checks +- **Dependency Audits**: Regular dependency updates +- **Access Reviews**: Regular access control reviews + +## Conclusion + +ServiceMaker simplifies the packaging and deployment of user-provided services, but accepting and running arbitrary code in your organization's platform introduces significant security and operational risks. These risks must be addressed through: + +- **Defense in Depth**: Multiple layers of security controls +- **Least Privilege**: Minimal permissions and access +- **Continuous Monitoring**: Real-time threat detection +- **Automated Security**: Security scanning and policy enforcement +- **Incident Response**: Preparedness for security incidents + +By implementing comprehensive security controls at every stage of the deployment pipeline, organizations can safely accept and run user-provided services while maintaining security and compliance. + +--- + +**Document Version**: 1.0 +**Last Updated**: 2024 +**Maintained By**: Platform Security Team + diff --git a/docs/SERVICE_COMPARISON.md b/docs/SERVICE_COMPARISON.md new file mode 100644 index 0000000..20ba4c7 --- /dev/null +++ b/docs/SERVICE_COMPARISON.md @@ -0,0 +1,527 @@ +# Node-Foxx Services vs Express + Arangojs Services + +This document explains the key differences between Node-Foxx services (using `@arangodb/node-foxx` libraries) and simple Express.js applications using Arangojs driver. + +## Overview + +Both approaches can build REST APIs that interact with ArangoDB, but they use different architectures and have different trade-offs. + +## Architecture Comparison + +### Node-Foxx Services + +**Components:** +- `@arangodb/node-foxx`: Framework providing router, context, and Foxx-specific features +- `@arangodb/node-foxx-launcher`: Orchestrator that manages services, worker threads, and HTTP server +- `@arangodb/arangodb`: Database client (used internally) + +**Architecture:** +``` +┌─────────────────────────────────────┐ +│ node-foxx-launcher (Main Thread) │ +│ - HTTP Server (Port 3000) │ +│ - Service Discovery │ +│ - Request Routing │ +│ - Token Management │ +└──────────────┬──────────────────────┘ + │ + ┌───────┴────────┐ + │ │ +┌──────▼──────┐ ┌──────▼──────┐ +│ Worker 1 │ │ Worker 2 │ +│ (Service A) │ │ (Service B) │ +│ │ │ │ +│ - node-foxx │ │ - node-foxx │ +│ - Router │ │ - Router │ +│ - Context │ │ - Context │ +└─────────────┘ └──────────────┘ +``` + +**Key Features:** +- Multi-service orchestration (multiple services on one port) +- Worker thread isolation (each service in separate thread) +- Automatic API documentation (Swagger) +- Service context and dependency injection +- Configuration management via `manifest.json` + +### Express + Arangojs Services + +**Components:** +- `express`: Web framework +- `arangojs`: Direct ArangoDB driver + +**Architecture:** +``` +┌─────────────────────────────┐ +│ Express Application │ +│ - HTTP Server │ +│ - Routes │ +│ - Middleware │ +│ - Direct Arangojs Access │ +└─────────────────────────────┘ +``` + +**Key Features:** +- Standalone application (one service per process) +- Standard Express.js patterns +- Direct database access +- Full control over application structure +- Manual API documentation (if needed) + +## Detailed Comparison + +### 1. Service Configuration + +**Node-Foxx:** +```json +// manifest.json +{ + "name": "my-service", + "version": "1.0.0", + "main": "index.js", + "engines": { "arangodb": "^3.0" } +} + +// services.json +[ + { + "mount": "/myservice", + "basePath": "." + } +] +``` + +**Express:** +```json +// package.json +{ + "name": "my-service", + "version": "1.0.0", + "main": "index.js", + "dependencies": { + "express": "^4.18.2", + "arangojs": "^10.1.2" + } +} +``` + +**Difference:** Express requires only `package.json`, no additional configuration files. + +### 2. Routing + +**Node-Foxx:** +```javascript +const createRouter = require('@arangodb/node-foxx/router'); +const { context } = require('@arangodb/node-foxx/locals'); + +const router = createRouter(); +context.use(router); + +router.get('/items', async (req, res) => { + const collection = context.collection('items'); + const items = await collection.all(); + res.send(items); +}) + .response('array', joi.array().items(joi.object()), 'List of items') + .summary('Get all items'); +``` + +**Express:** +```javascript +const express = require('express'); +const { Database } = require('arangojs'); + +const app = express(); +const db = new Database({ url: '...', auth: { token: '...' } }); + +app.get('/items', async (req, res) => { + const collection = db.collection('items'); + const cursor = await collection.all(); + const items = await cursor.all(); + res.json(items); +}); +``` + +**Difference:** +- Foxx: Uses Foxx router with built-in Swagger generation +- Express: Uses standard Express router, Swagger must be added manually + +### 3. Database Access + +**Node-Foxx:** +```javascript +const { context } = require('@arangodb/node-foxx/locals'); + +// Access via context (database connection managed by launcher) +const collection = context.collection('items'); +const db = context.database('mydb'); +``` + +**Express:** +```javascript +const { Database } = require('arangojs'); + +// Direct database connection (must manage yourself) +const db = new Database({ + url: process.env.ARANGO_DB_ENDPOINT, + auth: { token: process.env.ARANGODB_JWT_TOKEN }, + databaseName: process.env.ARANGO_DB_NAME +}); + +const collection = db.collection('items'); +``` + +**Difference:** +- Foxx: Database connection managed by launcher, accessed via context +- Express: Must create and manage database connections manually + +### 4. Multi-Service Support + +**Node-Foxx:** +```json +// services.json - Multiple services on one port +[ + { "mount": "/service1", "basePath": "./service1" }, + { "mount": "/service2", "basePath": "./service2" } +] +``` +- All services run on same port (e.g., 3000) +- Each service in separate worker thread +- Launcher routes requests to correct service + +**Express:** +- One service per application/process +- Each service needs its own port or reverse proxy +- No built-in multi-service orchestration + +**Difference:** Foxx supports multiple services on one port; Express requires separate processes/ports. + +### 5. API Documentation + +**Node-Foxx:** +```javascript +router.get('/items/:id', handler) + .pathParam('id', joi.string().required(), 'Item ID') + .response(200, joi.object(), 'Item details') + .summary('Get item by ID'); +``` +- Automatic Swagger generation from route definitions +- Built into router methods (`.body()`, `.response()`, `.pathParam()`) + +**Express:** +```javascript +/** + * @swagger + * /items/{id}: + * get: + * summary: Get item by ID + * parameters: + * - in: path + * name: id + * required: true + * schema: + * type: string + */ +app.get('/items/:id', handler); +``` +- Manual Swagger documentation using JSDoc comments +- Requires `swagger-jsdoc` and `swagger-ui-express` packages +- More verbose but more flexible + +**Difference:** Foxx auto-generates docs; Express requires manual documentation. + +### 6. Service Context + +**Node-Foxx:** +```javascript +const { context } = require('@arangodb/node-foxx/locals'); + +// Access service metadata +context.mount // "/myservice" +context.baseUrl // "http://localhost:3000/myservice" +context.manifest // manifest.json content +context.configuration // Service configuration +context.dependencies // Service dependencies +``` + +**Express:** +```javascript +// No built-in context +// Must manage configuration yourself +const config = require('./config'); +``` + +**Difference:** Foxx provides rich service context; Express requires manual configuration management. + +### 7. Setup/Teardown Scripts + +**Node-Foxx:** +```json +// manifest.json +{ + "scripts": { + "setup": "scripts/setup.js", + "teardown": "scripts/teardown.js" + } +} +``` +- Automatically executed by launcher during service installation/removal +- Access to service context and database + +**Express:** +```json +// package.json +{ + "scripts": { + "setup": "node scripts/setup.js" + } +} +``` +- Must be run manually +- Must manage database connection yourself + +**Difference:** Foxx scripts run automatically; Express scripts are manual. + +### 8. Error Handling + +**Node-Foxx:** +```javascript +router.get('/items/:id', async (req, res) => { + try { + const item = await collection.document(id); + res.send(item); + } catch (e) { + if (e.isArangoError && e.errorNum === 1202) { + res.throw(404, 'Item not found'); + } + throw e; + } +}); +``` +- Built-in error handling with `res.throw()` +- Automatic error response formatting + +**Express:** +```javascript +app.get('/items/:id', async (req, res, next) => { + try { + const item = await collection.document(id); + res.json(item); + } catch (e) { + if (e.isArangoError && e.errorNum === 1202) { + return res.status(404).json({ error: 'Item not found' }); + } + next(e); + } +}); + +// Error handling middleware +app.use((err, req, res, next) => { + res.status(err.status || 500).json({ error: err.message }); +}); +``` +- Standard Express error handling patterns +- Requires manual error middleware + +**Difference:** Foxx has built-in error handling; Express uses standard middleware patterns. + +### 9. Request/Response Objects + +**Node-Foxx:** +```javascript +// Synthetic request/response objects +router.get('/items', async (req, res) => { + req.body // Parsed request body + req.pathParams // Path parameters + req.queryParams // Query parameters + req.headers // Request headers + + res.send(data) // Send response (auto-formats) + res.json(data) // Send JSON + res.throw(404) // Throw error +}); +``` + +**Express:** +```javascript +// Standard Express request/response +app.get('/items', async (req, res) => { + req.body // Parsed request body + req.params // Path parameters + req.query // Query parameters + req.headers // Request headers + + res.json(data) // Send JSON + res.status(404) // Set status + res.send(data) // Send response +}); +``` + +**Difference:** Similar APIs, but Foxx uses synthetic objects; Express uses standard Express objects. + +### 10. Validation + +**Node-Foxx:** +```javascript +router.post('/items', handler) + .body(joi.object({ + name: joi.string().required(), + quantity: joi.number().optional() + }), 'Item to create') + .response(201, joi.object(), 'Created item'); +``` +- Built-in Joi validation +- Automatic Swagger schema generation + +**Express:** +```javascript +const itemSchema = Joi.object({ + name: Joi.string().required(), + quantity: Joi.number().optional() +}); + +app.post('/items', async (req, res) => { + const { error, value } = itemSchema.validate(req.body); + if (error) { + return res.status(400).json({ error: error.details[0].message }); + } + // ... handle request +}); +``` +- Manual validation +- Must handle validation errors yourself + +**Difference:** Foxx has built-in validation; Express requires manual validation. + +## When to Use Each + +### Use Node-Foxx When: + +✅ **Multi-service architecture**: Need multiple services on one port +✅ **Automatic API docs**: Want auto-generated Swagger documentation +✅ **Service isolation**: Need worker thread isolation +✅ **ArangoDB-native**: Building services specifically for ArangoDB +✅ **Configuration management**: Need built-in configuration system +✅ **Migration from ArangoDB Foxx**: Migrating existing Foxx services + +### Use Express + Arangojs When: + +✅ **Standard Express patterns**: Want familiar Express.js structure +✅ **Flexibility**: Need full control over application architecture +✅ **Standalone service**: Single service per application +✅ **Existing Express apps**: Migrating or extending Express applications +✅ **Simpler setup**: Don't need Foxx-specific features +✅ **Standard tooling**: Want to use standard Node.js/Express tooling + +## Migration Path + +### From Foxx to Express: + +1. **Remove Foxx dependencies**: + ```bash + npm uninstall @arangodb/node-foxx @arangodb/node-foxx-launcher + ``` + +2. **Add Express dependencies**: + ```bash + npm install express arangojs + ``` + +3. **Replace router**: + ```javascript + // Before (Foxx) + const router = createRouter(); + context.use(router); + + // After (Express) + const app = express(); + ``` + +4. **Update database access**: + ```javascript + // Before (Foxx) + const collection = context.collection('items'); + + // After (Express) + const db = new Database({ url: '...', auth: { token: '...' } }); + const collection = db.collection('items'); + ``` + +5. **Remove configuration files**: + - Delete `manifest.json` + - Delete `services.json` (if single service) + +6. **Update entrypoint**: + ```json + // package.json + { + "main": "index.js", + "scripts": { + "start": "node index.js" + } + } + ``` + +### From Express to Foxx: + +1. **Add Foxx dependencies**: + ```bash + npm install @arangodb/node-foxx @arangodb/node-foxx-launcher @arangodb/arangodb + ``` + +2. **Create manifest.json**: + ```json + { + "name": "my-service", + "version": "1.0.0", + "main": "index.js" + } + ``` + +3. **Create services.json**: + ```json + [ + { + "mount": "/myservice", + "basePath": "." + } + ] + ``` + +4. **Replace Express router with Foxx router**: + ```javascript + // Before (Express) + const app = express(); + app.get('/items', handler); + + // After (Foxx) + const router = createRouter(); + context.use(router); + router.get('/items', handler); + ``` + +5. **Update database access**: + ```javascript + // Before (Express) + const db = new Database({ ... }); + + // After (Foxx) + const collection = context.collection('items'); + ``` + +## Summary + +| Aspect | Node-Foxx | Express + Arangojs | +|--------|-----------|-------------------| +| **Complexity** | Higher (orchestration, worker threads) | Lower (standard Express) | +| **Multi-service** | ✅ Built-in | ❌ Requires separate processes | +| **API Docs** | ✅ Auto-generated | ⚠️ Manual (Swagger) | +| **Configuration** | ✅ Built-in (manifest.json) | ⚠️ Manual | +| **Flexibility** | ⚠️ Foxx-specific patterns | ✅ Full control | +| **Setup Scripts** | ✅ Automatic | ⚠️ Manual | +| **Dependencies** | More (Foxx libraries) | Fewer (Express + Arangojs) | +| **Learning Curve** | Steeper (Foxx concepts) | Gentler (standard Express) | +| **Use Case** | ArangoDB-native services | Standard web services | + +Both approaches are valid and serve different needs. Choose based on your requirements for multi-service support, API documentation, and architectural preferences. + diff --git a/scripts/prepareproject-express.sh b/scripts/prepareproject-express.sh new file mode 100644 index 0000000..f73c27f --- /dev/null +++ b/scripts/prepareproject-express.sh @@ -0,0 +1,79 @@ +#!/bin/bash +# This script installs only missing or incompatible project dependencies to the project's node_modules. +# Base node_modules at /home/user/node_modules is immutable and never copied. +# Uses check-base-dependencies.js to avoid duplicating packages that exist in base. + +set -e + +# We're in /project/{project-name} (WORKDIR) +# package.json is in the current directory +# node_modules will be created in the current directory +PROJECT_DIR=$(pwd) + +# Verify base node_modules exists (immutable, pre-scanned) +if [ ! -d "/home/user/node_modules" ]; then + echo "ERROR: Base node_modules not found at /home/user/node_modules" + exit 1 +fi + +echo "Base node_modules found at /home/user/node_modules (immutable)" + +# Install project dependencies if package.json exists +if [ -f "package.json" ]; then + echo "Analyzing dependencies against base node_modules..." + + # Check which packages need to be installed + CHECK_SCRIPT="/scripts/check-base-dependencies.js" + if [ ! -f "$CHECK_SCRIPT" ]; then + echo "ERROR: check-base-dependencies.js not found at $CHECK_SCRIPT" + exit 1 + fi + + # Run dependency check script + # Capture stdout (JSON) only, stderr (user messages) automatically goes to console + INSTALL_DATA=$(node "$CHECK_SCRIPT") + CHECK_RESULT=$? + + if [ $CHECK_RESULT -ne 0 ]; then + echo "ERROR: Failed to check base dependencies (exit code: $CHECK_RESULT)" + exit 1 + fi + + if [ -z "$INSTALL_DATA" ] || ! echo "$INSTALL_DATA" | grep -q '^{'; then + echo "ERROR: Could not parse dependency check output" + echo "Received: $INSTALL_DATA" + exit 1 + fi + PACKAGES_TO_INSTALL=$(echo "$INSTALL_DATA" | node -e "const data=JSON.parse(require('fs').readFileSync(0,'utf8')); console.log(data.packagesToInstall.map(p => p.split('@')[0]).join(' '))") + + # Count packages + TOTAL_DEPS=$(echo "$INSTALL_DATA" | node -e "const data=JSON.parse(require('fs').readFileSync(0,'utf8')); console.log(data.totalDependencies)") + FROM_BASE=$(echo "$INSTALL_DATA" | node -e "const data=JSON.parse(require('fs').readFileSync(0,'utf8')); console.log(data.packagesFromBase)") + TO_INSTALL_COUNT=$(echo "$INSTALL_DATA" | node -e "const data=JSON.parse(require('fs').readFileSync(0,'utf8')); console.log(data.packagesToInstall.length)") + + echo "" + echo "Dependency summary:" + echo " Total dependencies: $TOTAL_DEPS" + echo " Available in base: $FROM_BASE" + echo " To install: $TO_INSTALL_COUNT" + echo "" + + # Install only packages that are missing or incompatible + if [ -n "$PACKAGES_TO_INSTALL" ] && [ "$PACKAGES_TO_INSTALL" != "" ]; then + echo "Installing missing/incompatible packages: $PACKAGES_TO_INSTALL" + # Install packages individually to avoid installing everything + for package_spec in $(echo "$INSTALL_DATA" | node -e "const data=JSON.parse(require('fs').readFileSync(0,'utf8')); console.log(data.packagesToInstall.join(' '))"); do + npm install --production --no-save "$package_spec" + done + echo "✓ Project-specific dependencies installed" + else + echo "✓ All dependencies satisfied by base node_modules (no installation needed)" + # Create empty node_modules directory if it doesn't exist (for consistency) + mkdir -p node_modules + fi +else + echo "No package.json found, skipping dependency installation" +fi + +echo "Express application prepared successfully" + diff --git a/src/main.rs b/src/main.rs index f2ff9f1..381672a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -58,6 +58,10 @@ const SCRIPT_FILES: &[ScriptFile] = &[ path: "prepareproject-nodejs.sh", content: include_str!("../scripts/prepareproject-nodejs.sh"), }, + ScriptFile { + path: "prepareproject-express.sh", + content: include_str!("../scripts/prepareproject-express.sh"), + }, ScriptFile { path: "check-base-dependencies.js", content: include_str!("../scripts/check-base-dependencies.js"), @@ -154,6 +158,29 @@ fn main() -> Result<(), Box> { args.base_image = Some(DEFAULT_PYTHON_BASE_IMAGE.to_string()); } } + "express" => { + // Express application - standalone, no services.json or manifest.json + // Try to get name from package.json if not provided + if args.name.is_none() + && let Ok(name) = read_name_from_package_json(project_home) + { + args.name = Some(name); + } + + // Try to auto-detect entrypoint from package.json "main" field or "start" script + if args.entrypoint.is_none() { + if let Ok(Some(entrypoint)) = detect_express_entrypoint(project_home) { + args.entrypoint = Some(entrypoint); + } else { + args.entrypoint = Some("index.js".to_string()); + } + } + + // Set default base image for Node.js if not explicitly set + if !base_image_explicitly_set { + args.base_image = Some(DEFAULT_NODEJS_BASE_IMAGE.to_string()); + } + } "foxx" => { // Multi-service structure (has services.json) // Try to get name from package.json if not provided @@ -241,8 +268,18 @@ fn main() -> Result<(), Box> { // Copy scripts to temp directory with executable permissions copy_scripts_to_temp(&temp_dir)?; - // Handle Node.js single service directory - copy directly and generate services.json - let (service_name_opt, project_dir) = if project_type == "foxx-service" { + // Handle Express and Foxx service directories + let (service_name_opt, project_dir) = if project_type == "express" { + // Express app - copy directly, no services.json needed + let project_dest = temp_dir.join(initial_project_dir); + println!( + "Copying Express app from {} to {}", + project_home.display(), + project_dest.display() + ); + copy_dir_recursive(project_home, &project_dest)?; + (None, initial_project_dir.to_string()) + } else if project_type == "foxx-service" { let service_name = initial_project_dir; // Copy service directory directly (no wrapper folder) @@ -274,6 +311,12 @@ fn main() -> Result<(), Box> { (None, initial_project_dir.to_string()) }; + // Read environment variables from .env.example if it exists + let env_vars = read_env_example(project_home)?; + if !env_vars.is_empty() { + println!("Found {} environment variable(s) in .env.example", env_vars.len()); + } + // Choose Dockerfile template and modify based on project type let modified_dockerfile = match project_type.as_str() { "python" => { @@ -287,6 +330,19 @@ fn main() -> Result<(), Box> { entrypoint, port, &python_version, + &env_vars, + ) + } + "express" => { + let entrypoint = args.entrypoint.as_ref().unwrap(); + let dockerfile_template = include_str!("../Dockerfile.express.template"); + modify_dockerfile_express( + dockerfile_template, + base_image, + &project_dir, + entrypoint, + port, + &env_vars, ) } "foxx" | "foxx-service" => { @@ -297,6 +353,7 @@ fn main() -> Result<(), Box> { &project_dir, port, service_name_opt.as_deref(), + &env_vars, ) } _ => return Err("Unsupported project type".into()), @@ -433,7 +490,7 @@ fn main() -> Result<(), Box> { println!("Version from pyproject.toml: {}", ver); (name, ver) } - "foxx" | "foxx-service" => { + "express" | "foxx" | "foxx-service" => { let (name, ver) = read_service_info_from_package_json(project_home)?; println!("Service name from package.json: {}", name); println!("Version from package.json: {}", ver); @@ -524,13 +581,73 @@ fn modify_dockerfile_python( entrypoint: &str, port: u16, python_version: &str, + env_vars: &[(String, String)], ) -> String { - template + let mut result = template .replace("{BASE_IMAGE}", base_image) .replace("{PROJECT_DIR}", project_dir) .replace("{PORT}", &port.to_string()) .replace("{ENTRYPOINT}", entrypoint) - .replace("{PYTHON_VERSION}", python_version) + .replace("{PYTHON_VERSION}", python_version); + + // Add environment variables if any + if !env_vars.is_empty() { + let env_lines: Vec = env_vars + .iter() + .map(|(key, value)| format!("ENV {}={}", key, value)) + .collect(); + let env_block = format!("\n{}", env_lines.join("\n")); + + // Insert after WORKDIR line + if let Some(pos) = result.find("WORKDIR") + && let Some(newline_pos) = result[pos..].find('\n') { + let insert_pos = pos + newline_pos + 1; + result.insert_str(insert_pos, &env_block); + } + } + + result +} + +fn modify_dockerfile_express( + template: &str, + base_image: &str, + project_dir: &str, + entrypoint: &str, + port: u16, + env_vars: &[(String, String)], +) -> String { + // Express app structure: + // - COPY copies the project directory directly + // - WORKDIR is /project/{project-dir} + // - node_modules is in /project/{project-dir}/node_modules + let node_path = format!("/project/{}/node_modules:/home/user/node_modules", project_dir); + + let mut result = template + .replace("{BASE_IMAGE}", base_image) + .replace("{PROJECT_DIR}", project_dir) + .replace("{WORKDIR}", project_dir) + .replace("{ENTRYPOINT}", entrypoint) + .replace("{PORT}", &port.to_string()) + .replace("{NODE_PATH}", &node_path); + + // Add environment variables if any + if !env_vars.is_empty() { + let env_lines: Vec = env_vars + .iter() + .map(|(key, value)| format!("ENV {}={}", key, value)) + .collect(); + let env_block = format!("\n{}", env_lines.join("\n")); + + // Insert after NODE_PATH ENV line + if let Some(pos) = result.find("ENV NODE_PATH") + && let Some(newline_pos) = result[pos..].find('\n') { + let insert_pos = pos + newline_pos + 1; + result.insert_str(insert_pos, &env_block); + } + } + + result } fn modify_dockerfile_nodejs( @@ -539,6 +656,7 @@ fn modify_dockerfile_nodejs( project_dir: &str, port: u16, _service_name: Option<&str>, + env_vars: &[(String, String)], ) -> String { // Simplified structure: // - COPY copies the service directory directly (services.json is inside) @@ -546,26 +664,127 @@ fn modify_dockerfile_nodejs( // - node_modules is in /project/{service-name}/node_modules let node_path = format!("/project/{}/node_modules:/home/user/node_modules", project_dir); - template + let mut result = template .replace("{BASE_IMAGE}", base_image) .replace("{PROJECT_DIR}", project_dir) .replace("{WORKDIR}", project_dir) .replace("{PORT}", &port.to_string()) - .replace("{NODE_PATH}", &node_path) + .replace("{NODE_PATH}", &node_path); + + // Add environment variables if any + if !env_vars.is_empty() { + let env_lines: Vec = env_vars + .iter() + .map(|(key, value)| format!("ENV {}={}", key, value)) + .collect(); + let env_block = format!("\n{}", env_lines.join("\n")); + + // Insert after NODE_PATH ENV line + if let Some(pos) = result.find("ENV NODE_PATH") + && let Some(newline_pos) = result[pos..].find('\n') { + let insert_pos = pos + newline_pos + 1; + result.insert_str(insert_pos, &env_block); + } + } + + result +} + +fn read_env_example(project_home: &Path) -> Result, Box> { + let env_example_path = project_home.join(".env.example"); + + if !env_example_path.exists() { + return Ok(Vec::new()); + } + + let content = fs::read_to_string(&env_example_path)?; + let mut env_vars = Vec::new(); + + for line in content.lines() { + let line = line.trim(); + + // Skip empty lines and comments + if line.is_empty() || line.starts_with('#') { + continue; + } + + // Parse KEY=VALUE format + if let Some(equal_pos) = line.find('=') { + let key = line[..equal_pos].trim().to_string(); + let mut value = line[equal_pos + 1..].trim().to_string(); + + // Remove quotes if present (handles both single and double quotes) + if (value.starts_with('"') && value.ends_with('"')) + || (value.starts_with('\'') && value.ends_with('\'')) { + value = value[1..value.len() - 1].to_string(); + } + + // Skip if key is empty + if !key.is_empty() { + // If value contains spaces or special characters, quote it for Docker ENV + let final_value = if value.contains(' ') || value.contains('$') || value.contains('\\') { + format!("\"{}\"", value.replace('"', "\\\"")) + } else { + value + }; + env_vars.push((key, final_value)); + } + } + } + + Ok(env_vars) +} + +fn detect_express_entrypoint(project_home: &Path) -> Result, Box> { + let package_json_path = project_home.join("package.json"); + + if !package_json_path.exists() { + return Ok(None); + } + + let content = fs::read_to_string(&package_json_path)?; + let value: serde_json::Value = serde_json::from_str(&content)?; + + // Try to get from "main" field + if let Some(main) = value.get("main").and_then(|m| m.as_str()) { + return Ok(Some(main.to_string())); + } + + // Try to extract from "start" script + if let Some(scripts) = value.get("scripts") + && let Some(start) = scripts.get("start").and_then(|s| s.as_str()) { + // Extract the script name from "node index.js" or "node app.js" + if let Some(script_name) = start.strip_prefix("node ") { + return Ok(Some(script_name.trim().to_string())); + } + } + + Ok(None) } fn detect_project_type(project_home: &Path) -> Result> { let pyproject = project_home.join("pyproject.toml"); let package_json = project_home.join("package.json"); let services_json = project_home.join("services.json"); + let manifest_json = project_home.join("manifest.json"); if pyproject.exists() { Ok("python".to_string()) - } else if package_json.exists() && services_json.exists() { - Ok("foxx".to_string()) } else if package_json.exists() { - // Single service directory - needs wrapper structure - Ok("foxx-service".to_string()) + // Check if it's an Express app (has express dependency, no services.json, no manifest.json) + if !services_json.exists() && !manifest_json.exists() + && let Ok(is_express) = is_express_app(project_home) + && is_express { + return Ok("express".to_string()); + } + + // Check for Foxx services + if services_json.exists() { + Ok("foxx".to_string()) + } else { + // Single service directory - needs wrapper structure + Ok("foxx-service".to_string()) + } } else { Err(format!( "Could not detect project type. Expected pyproject.toml (Python) or package.json (Node.js) in: {}", @@ -575,6 +794,29 @@ fn detect_project_type(project_home: &Path) -> Result Result> { + let package_json_path = project_home.join("package.json"); + + if !package_json_path.exists() { + return Ok(false); + } + + let content = fs::read_to_string(&package_json_path)?; + let value: serde_json::Value = serde_json::from_str(&content)?; + + // Check if express is in dependencies or devDependencies + let has_express = value + .get("dependencies") + .and_then(|deps| deps.get("express")) + .is_some() + || value + .get("devDependencies") + .and_then(|deps| deps.get("express")) + .is_some(); + + Ok(has_express) +} + fn generate_services_json(_service_name: &str) -> String { // basePath is "." because services.json is in the same directory as the service // and node-foxx runs from that directory (WORKDIR) diff --git a/testprojects/shoppinglist-express/.env.example b/testprojects/shoppinglist-express/.env.example new file mode 100644 index 0000000..53a5d6f --- /dev/null +++ b/testprojects/shoppinglist-express/.env.example @@ -0,0 +1,4 @@ +PORT=3000 +NODE_ENV=development +ARANGO_DB_ENDPOINT=https://arangodb-platform-dev.pilot.arangodb.com +ARANGO_DB_NAME=FoxxServices diff --git a/testprojects/shoppinglist-express/README.md b/testprojects/shoppinglist-express/README.md new file mode 100644 index 0000000..da86268 --- /dev/null +++ b/testprojects/shoppinglist-express/README.md @@ -0,0 +1,53 @@ +# Shopping List Express - Mini Version + +A minimal, straightforward shopping list API built with Express.js and ArangoDB. + +## Project Structure + +``` +shoppinglist-express/ +├── index.js # Main app with all routes +├── schemas.js # Joi validation schemas +├── swagger.js # Swagger configuration +├── package.json # Dependencies +├── .env # Environment variables +└── README.md # This file +``` + +## Setup + +1. Install dependencies: +```bash +npm install +``` + +2. Update `.env` with your configuration: +``` +PORT=3000 +NODE_ENV=development +``` + +3. Connect to ArangoDB (in `index.js`, attach your db connection to `req.db`) + +4. Start the server: +```bash +npm start +``` + +## API Endpoints + +- `GET /health` - Health check +- `GET /docs` - Swagger documentation +- `POST /api/items` - Create item +- `GET /api/items` - Get all items +- `GET /api/items/:key` - Get item by key +- `DELETE /api/items/:key` - Delete item + +## Features + +✅ Simple and straightforward structure +✅ Minimal folder organization +✅ Swagger documentation included +✅ Input validation with Joi +✅ Error handling middleware +✅ Graceful shutdown diff --git a/testprojects/shoppinglist-express/index.js b/testprojects/shoppinglist-express/index.js new file mode 100644 index 0000000..7d33a1c --- /dev/null +++ b/testprojects/shoppinglist-express/index.js @@ -0,0 +1,226 @@ +require('dotenv').config(); + +const express = require('express'); +const { Database } = require('arangojs'); +const swaggerUi = require('swagger-ui-express'); +const swaggerSpec = require('./swagger'); +const itemSchema = require('./schemas'); + +const app = express(); +const PORT = process.env.PORT || 3000; +const DOCUMENT_NOT_FOUND = 1202; +const collectionName = 'shoppinglist'; + +// Middleware +app.use(express.json()); + +// Swagger UI +app.use('/docs', swaggerUi.serve, swaggerUi.setup(swaggerSpec, { + customCss: '.swagger-ui .topbar { display: none }', + customSiteTitle: 'Shopping List API', +})); + +// Health check +app.get('/health', (req, res) => { + res.json({ status: 'OK' }); +}); + +// Database middleware (simulated - connect to ArangoDB) +const dbMiddleware = (req, res, next) => { + const authHeader = req.headers.authorization || req.headers.Authorization; + const dbEndpoint = process.env.ARANGO_DB_ENDPOINT || 'http://127.0.0.1:8529'; + const dbName = process.env.ARANGO_DB_NAME || '_system'; + let token = null; + + if (authHeader) { + const parts = authHeader.split(' '); + token = parts.length === 2 && parts[0] === 'Bearer' ? parts[1] : authHeader; + } + + if (!token) { + return res.status(401).json({ error: 'Authorization header with JWT token is required' }); + } + + try { + const db = new Database({ + url: dbEndpoint, + auth: { token }, + databaseName: dbName + }); + + req.db = db; + next(); + } catch (error) { + console.error('Database connection error:', error.message); + return res.status(500).json({ error: 'Database connection error', message: error.message }); + } +}; + +app.use('/api', dbMiddleware); + +/** + * @swagger + * /api/items: + * post: + * summary: Create a new shopping list item + * tags: [Items] + * requestBody: + * required: true + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/Item' + * responses: + * 201: + * description: Item created + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/Item' + * 400: + * description: Validation error + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/Error' + */ +app.post('/api/items', async (req, res, next) => { + try { + const { error, value } = itemSchema.validate(req.body); + if (error) { + return res.status(400).json({ error: error.details[0].message }); + } + + const collection = req.db.collection(collectionName); + const meta = await collection.save(value); + const savedItem = await collection.document(meta._key); + res.status(201).json(savedItem); + } catch (err) { + next(err); + } +}); + +/** + * @swagger + * /api/items: + * get: + * summary: Get all items + * tags: [Items] + * responses: + * 200: + * description: List of all items + * content: + * application/json: + * schema: + * type: array + * items: + * $ref: '#/components/schemas/Item' + */ +app.get('/api/items', async (req, res, next) => { + try { + const cursor = await req.db.query('FOR item IN @@collection RETURN item', { + '@collection': collectionName, + }); + const items = await cursor.all(); + res.json(items); + } catch (err) { + next(err); + } +}); + +/** + * @swagger + * /api/items/{key}: + * get: + * summary: Get item by key + * tags: [Items] + * parameters: + * - in: path + * name: key + * required: true + * schema: + * type: string + * responses: + * 200: + * description: Item found + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/Item' + * 404: + * description: Item not found + */ +app.get('/api/items/:key', async (req, res, next) => { + try { + const { key } = req.params; + if (!key?.trim()) { + return res.status(400).json({ error: 'Invalid key' }); + } + + const collection = req.db.collection(collectionName); + const item = await collection.document(key); + res.json(item); + } catch (err) { + if (err.isArangoError && err.errorNum === DOCUMENT_NOT_FOUND) { + return res.status(404).json({ error: 'Item not found' }); + } + next(err); + } +}); + +/** + * @swagger + * /api/items/{key}: + * delete: + * summary: Delete item by key + * tags: [Items] + * parameters: + * - in: path + * name: key + * required: true + * schema: + * type: string + * responses: + * 204: + * description: Item deleted + * 404: + * description: Item not found + */ +app.delete('/api/items/:key', async (req, res, next) => { + try { + const { key } = req.params; + if (!key?.trim()) { + return res.status(400).json({ error: 'Invalid key' }); + } + + const collection = req.db.collection(collectionName); + await collection.remove(key); + res.status(204).send(); + } catch (err) { + if (err.isArangoError && err.errorNum === DOCUMENT_NOT_FOUND) { + return res.status(404).json({ error: 'Item not found' }); + } + next(err); + } +}); + +// Error handling +app.use((err, req, res, next) => { + console.error('Error:', process.env.NODE_ENV === 'development' ? err : err.message); + res.status(err.status || 500).json({ + error: err.message || 'Internal server error', + }); +}); + +// Start server +const server = app.listen(PORT, () => { + console.log(`Server running on port ${PORT}`); +}); + +// Graceful shutdown +process.on('SIGTERM', () => { + console.log('SIGTERM received, shutting down gracefully...'); + server.close(() => process.exit(0)); +}); + +module.exports = app; diff --git a/testprojects/shoppinglist-express/package.json b/testprojects/shoppinglist-express/package.json new file mode 100644 index 0000000..c85bdc1 --- /dev/null +++ b/testprojects/shoppinglist-express/package.json @@ -0,0 +1,19 @@ +{ + "name": "shoppinglist-express", + "version": "1.0.0", + "description": "A minimal shopping list service using Express.js and Arangojs", + "main": "index.js", + "scripts": { + "start": "node index.js" + }, + "author": "Yaswanth", + "license": "Apache-2.0", + "dependencies": { + "express": "^4.18.2", + "arangojs": "^10.1.2", + "joi": "^17.13.3", + "dotenv": "^16.4.5", + "swagger-jsdoc": "^6.2.8", + "swagger-ui-express": "^5.0.0" + } +} \ No newline at end of file diff --git a/testprojects/shoppinglist-express/schemas.js b/testprojects/shoppinglist-express/schemas.js new file mode 100644 index 0000000..9b44072 --- /dev/null +++ b/testprojects/shoppinglist-express/schemas.js @@ -0,0 +1,10 @@ +const Joi = require('joi'); + +// Item validation schema +const itemSchema = Joi.object({ + name: Joi.string().required(), + quantity: Joi.number().optional(), + completed: Joi.boolean().optional().default(false), +}); + +module.exports = itemSchema; diff --git a/testprojects/shoppinglist-express/swagger.js b/testprojects/shoppinglist-express/swagger.js new file mode 100644 index 0000000..b6d66c8 --- /dev/null +++ b/testprojects/shoppinglist-express/swagger.js @@ -0,0 +1,52 @@ +const swaggerJsdoc = require('swagger-jsdoc'); + +const options = { + definition: { + openapi: '3.0.0', + info: { + title: 'Shopping List API', + version: '1.0.0', + description: 'A minimal shopping list service', + contact: { name: 'API Support' }, + }, + servers: [ + { + url: `http://localhost:${process.env.PORT || 3000}`, + description: 'Local Development', + }, + ], + components: { + securitySchemes: { + bearerAuth: { + type: 'http', + scheme: 'bearer', + bearerFormat: 'JWT', + }, + }, + schemas: { + Item: { + type: 'object', + properties: { + _key: { type: 'string', example: '12345' }, + _id: { type: 'string', example: 'shoppinglist/12345' }, + _rev: { type: 'string', example: '_abc123' }, + name: { type: 'string', example: 'Milk' }, + quantity: { type: 'number', example: 2 }, + completed: { type: 'boolean', example: false }, + }, + }, + Error: { + type: 'object', + properties: { + error: { type: 'string' }, + message: { type: 'string' }, + }, + }, + }, + }, + security: [{ bearerAuth: [] }], + }, + apis: ['./index.js'], +}; + +module.exports = swaggerJsdoc(options);