Not running in docker container #5
Assignees
Labels
Comments
|
Hi @anuj9196 At the moment i have no idea what it could be the reason, at least i will try your dockerfile. Can you share the changes you made to |
|
@kalwalt These are the changes
node app.js -i /static/media/image.png -o /result/outputThe const path = require("path");
const fs = require('fs');
const sharp = require('sharp');
const { prompt } = require('enquirer');
var Module = require('./build/NftMarkerCreator_wasm.js');
// GLOBAL VARs
var params = [
];
var validImageExt = [".jpg", ".jpeg", ".png"];
var srcImage;
var outputPath = 'output/';
var buffer;
var foundInputPath = {
b: false,
i: -1
}
var foundOutputPath = {
b: false,
i: -1
}
var noConf = false;
var withDemo = false;
var onlyConfidence = false;
var isZFT = false;
var imageData = {
sizeX: 0,
sizeY: 0,
nc: 0,
dpi: 0,
array: []
}
Module.onRuntimeInitialized = async function () {
console.log('arguments...: ', process.argv);
for (let j = 2; j < process.argv.length; j++) {
if (process.argv[j].indexOf('-i') !== -1 || process.argv[j].indexOf('-I') !== -1) {
foundInputPath.b = true;
foundInputPath.i = j + 1;
j++;
} else if (process.argv[j] === "-NoConf") {
noConf = true;
} else if (process.argv[j] === "-Demo") {
withDemo = true;
} else if (process.argv[j] === "-zft") {
isZFT = true;
} else if (process.argv[j] === "-onlyConfidence") {
onlyConfidence = true;
} else if (process.argv[j].indexOf('-o') !== -1 || process.argv[j].indexOf('-O') !== -1) {
foundOutputPath.b = true;
foundOutputPath.i = j + 1;
j++;
} else {
console.log(process.argv[j]);
params.push(process.argv[j]);
}
}
if (!foundInputPath.b) {
const response = await prompt({
type: 'input',
name: 'inputPath',
message: 'Image path not present, to continue provide a path to image:'
});
srcImage = response.inputPath;
} else {
srcImage = process.argv[foundInputPath.i];
}
if (!srcImage) {
console.log("\nERROR: No image in INPUT command!\n e.g:(-i /PATH/TO/IMAGE)\n");
process.exit(1);
} else {
console.log('checking file path...: ', srcImage);
if (!srcImage.startsWith('/')) {
// Relative path
srcImage = path.join(__dirname, srcImage);
}
console.log('image path is: ', srcImage);
// srcImage = path.join(__dirname, process.argv[foundInputPath.i]);
}
if (foundOutputPath.b) {
outputPath = process.argv[foundOutputPath.i];
if (!outputPath.startsWith('/')) {
// relative path
outputPath = path.join(__dirname, outputPath);
// outputPath = '/' + outputPath;
}
if (!outputPath.endsWith('/')) {
outputPath += '/';
}
console.log('Set output path: ' + outputPath);
}
let fileNameWithExt = path.basename(srcImage);
let fileName = path.parse(fileNameWithExt).name;
let extName = path.parse(fileNameWithExt).ext;
let foundExt = false;
for (let ext in validImageExt) {
if (extName.toLowerCase() === validImageExt[ext]) {
foundExt = true;
break;
}
}
if (!foundExt) {
console.log("\nERROR: Invalid image TYPE!\n Valid types:(jpg,JPG,jpeg,JPEG,png,PNG)\n");
process.exit(1);
}
if (!fs.existsSync(srcImage)) {
console.log("\nERROR: Not possible to read image, probably invalid image PATH!\n");
process.exit(1);
} else {
buffer = fs.readFileSync(srcImage);
}
console.log('Check output path: ' + outputPath);
if (!fs.existsSync(outputPath)) {
fs.mkdirSync(outputPath);
}
if (extName.toLowerCase() === ".jpg" || extName.toLowerCase() === ".jpeg" || extName.toLowerCase() === ".png") {
await processImage(buffer);
}
let confidence = calculateQuality();
let txt = " - - - - - ";
if (confidence.l != 0) {
let str = txt.split(" ");
str.pop();
str.shift();
for (let i = 0; i < parseInt(confidence.l); i++) {
str[i] = " *";
}
str.push(" ");
txt = str.join("");
}
if (onlyConfidence) {
console.log("%f", confidence.l);
process.exit(0);
}
console.log("\nConfidence level: [" + txt + "] %f/5 || Entropy: %f || Current max: 5.17 min: 4.6\n", confidence.l, confidence.e)
if (noConf) {
await askToContinue();
}
let paramStr = params.join(' ');
let StrBuffer = Module._malloc(paramStr.length + 1);
Module.stringToUTF8(paramStr, StrBuffer);
console.log('Write Success');
let heapSpace = Module._malloc(imageData.array.length * imageData.array.BYTES_PER_ELEMENT);
Module.HEAPU8.set(imageData.array, heapSpace);
console.log('Setting Heap Success.. Continue to Create ImageSet..');
Module._createNftDataSet(heapSpace, imageData.dpi, imageData.sizeX, imageData.sizeY, imageData.nc, StrBuffer)
console.log('Create NFT Dataset complete...')
Module._free(heapSpace);
Module._free(StrBuffer);
let filenameIset = "tempFilename.iset";
let filenameFset = "tempFilename.fset";
let filenameFset3 = "tempFilename.fset3";
let ext = ".iset";
let ext2 = ".fset";
let ext3 = ".fset3";
let content = Module.FS.readFile(filenameIset);
let contentFset = Module.FS.readFile(filenameFset);
let contentFset3 = Module.FS.readFile(filenameFset3);
if (isZFT) {
console.log("CREATING ZFT FILE");
let iset = Buffer.from(content.buffer);
let fset = Buffer.from(contentFset.buffer);
let fset3 = Buffer.from(contentFset3.buffer);
let obj = {
iset: iset.toString('hex'),
fset: fset.toString('hex'),
fset3: fset3.toString('hex')
}
let strObj = JSON.stringify(obj);
let StrBufferZip = Module._malloc(strObj.length + 1);
Module.stringToUTF8(strObj, StrBufferZip);
Module._compressZip(StrBufferZip, strObj.length);
let contentBin = Module.FS.readFile("tempBinFile.bin");
// fs.writeFileSync(path.join(__dirname, '/output/') + fileName + ".zft", contentBin);
fs.writeFileSync(outputPath + fileName + ".zft", contentBin);
Module._free(StrBufferZip);
if (withDemo) {
console.log("\nFinished marker creation!\nNow configuring demo! \n")
const markerDir = path.join(__dirname, './../demo/public/marker/');
if (!fs.existsSync(markerDir)) {
fs.mkdirSync(markerDir);
}
let demoHTML = fs.readFileSync("./demo/nft.html").toString('utf8').split("\n");
addNewMarker(demoHTML, fileName);
let newHTML = demoHTML.join('\n');
fs.writeFileSync("./demo/nft.html", newHTML, { encoding: 'utf8', flag: 'w' });
const files = fs.readdirSync(markerDir);
for (const file of files) {
fs.unlink(path.join(markerDir, file), err => {
if (err) throw err;
});
}
fs.writeFileSync(markerDir + fileName + ".zft", contentBin);
console.log("Finished!\nTo run demo use: 'npm run demo'");
}
} else {
console.log("CREATING ISET, FSET AND FSET3 FILES");
// fs.writeFileSync(path.join(__dirname, outputPath) + fileName + ext, content);
// fs.writeFileSync(path.join(__dirname, outputPath) + fileName + ext2, contentFset);
// fs.writeFileSync(path.join(__dirname, outputPath) + fileName + ext3, contentFset3);
fs.writeFileSync(outputPath + fileName + ext, content);
fs.writeFileSync(outputPath + fileName + ext2, contentFset);
fs.writeFileSync(outputPath + fileName + ext3, contentFset3);
if (withDemo) {
console.log("\nFinished marker creation!\nNow configuring demo! \n")
const markerDir = path.join(__dirname, './../demo/public/marker/');
if (!fs.existsSync(markerDir)) {
fs.mkdirSync(markerDir);
}
let demoHTML = fs.readFileSync("./../demo/nft.html").toString('utf8').split("\n");
addNewMarker(demoHTML, fileName);
let newHTML = demoHTML.join('\n');
fs.writeFileSync("./../demo/nft.html", newHTML, { encoding: 'utf8', flag: 'w' });
const files = fs.readdirSync(markerDir);
for (const file of files) {
fs.unlink(path.join(markerDir, file), err => {
if (err) throw err;
});
}
fs.writeFileSync(markerDir + fileName + ext, content);
fs.writeFileSync(markerDir + fileName + ext2, contentFset);
fs.writeFileSync(markerDir + fileName + ext3, contentFset3);
console.log("Finished!\nTo run demo use: 'npm run demo'");
}
}
process.exit(0);
}
async function processImage(buf) {
const image = sharp(buf);
await image.metadata()
.then(async metadata => {
if (metadata.density) {
imageData.dpi = metadata.density;
} else {
console.log("\nWARNING: No DPI value found! Using 150 as default value!\n");
imageData.dpi = 150;
}
if(metadata.width){
imageData.sizeX = metadata.width;
} else {
await metadataWidth();
}
if(metadata.height){
imageData.sizeY = metadata.height;
} else {
await metadataHeigth();
}
if(metadata.channels){
imageData.nc = metadata.channels;
} else {
await metadataChannels();
}
return image
.raw()
.toBuffer()
})
.then(data => {
let dt = data.buffer
let verifyColorSpace = detectColorSpace(dt);
if (verifyColorSpace === 1) {
console.log("Color Space is 1 channel!");
} else if (verifyColorSpace === 3) {
console.log("Color Space is 3 channels!");
}
let uint = new Uint8Array(dt);
if(imageData.nc === verifyColorSpace){
console.log("Color Space is equal to metadata.channels!");
}else{
console.log("Color Space is not equal to metadata.channels!");
//process.exit(1);
}
imageData.nc = verifyColorSpace;
imageData.array = uint;
})
.catch(function (err) {
console.error("Error extracting metadata: " + err);
process.exit(1);
});
}
async function extractMetadata(buf) {
return await sharp(buf).metadata()
.then(function (metadata) {
if (metadata.density) {
imageData.dpi = metadata.density;
} else {
console.log("\nWARNING: No DPI value found! Using 150 as default value!\n");
imageData.dpi = 150;
}
imageData.sizeX = metadata.width;
imageData.sizeY = metadata.height;
imageData.nc = metadata.channels;
}).catch(function (err) {
console.error("Error extracting metadata: " + err);
process.exit(1);
});
}
function detectColorSpace(arr) {
let target = parseInt(arr.length / 4);
let counter = 0;
for (let j = 0; j < arr.length; j += 4) {
let r = arr[j];
let g = arr[j + 1];
let b = arr[j + 2];
if (r === g && r === b) {
counter++;
}
}
if (target === counter) {
return 1;
} else {
return 3;
}
}
function rgbaToRgb(arr) {
let newArr = [];
let BGColor = {
R: 255,
G: 255,
B: 255
}
for (let i = 0; i < arr.length; i += 4) {
let r = parseInt(255 * (((1 - arr[i + 3]) * BGColor.R) + (arr[i + 3] * arr[i])));
let g = parseInt(255 * (((1 - arr[i + 3]) * BGColor.G) + (arr[i + 3] * arr[i + 1])));
let b = parseInt(255 * (((1 - arr[i + 3]) * BGColor.B) + (arr[i + 3] * arr[i + 2])));
newArr.push(r);
newArr.push(g);
newArr.push(b);
}
return newArr;
}
function calculateQuality() {
let gray = toGrayscale(imageData.array);
let hist = getHistogram(gray);
let ent = 0;
let totSize = imageData.sizeX * imageData.sizeY;
for (let i = 0; i < 255; i++) {
if (hist[i] > 0) {
let temp = (hist[i] / totSize) * (Math.log(hist[i] / totSize));
ent += temp;
}
}
let entropy = (-1 * ent).toFixed(2);
let oldRange = (5.17 - 4.6);
let newRange = (5 - 0);
let level = (((entropy - 4.6) * newRange) / oldRange);
if (level > 5) {
level = 5;
} else if (level < 0) {
level = 0;
}
return { l: level.toFixed(2), e: entropy };
}
function toGrayscale(arr) {
let gray = [];
for (let i = 0; i < arr.length; i += 3) {
let avg = (arr[i] + arr[i + 1] + arr[i + 2]) / 3;
gray.push(parseInt(avg));
}
return gray;
}
function getHistogram(arr) {
let hist = [256];
for (let i = 0; i < arr.length; i++) {
hist[i] = 0;
}
for (let i = 0; i < arr.length; i++) {
hist[arr[i]]++;
}
return hist;
}
function addNewMarker(text, name) {
for (let i = 0; i < text.length; i++) {
if (text[i].trim().includes("<script>MARKER_NAME =")) {
text[i] = "<script>MARKER_NAME = '" + name + "'</script>"
break;
}
}
}
async function askToContinue() {
const response = await prompt({
type: 'input',
name: 'answer',
message: 'Do you want to continue? (Y/N)\n'
});
if (response.answer == "n") {
console.log("\nProcess finished by the user! \n");
process.exit(1);
}
}
async function metadataWidth() {
const responseToProceed = await prompt({
type: 'input',
name: 'answer',
message: 'Metadata width not present do you want to inform it? (Y/N)\n'
});
if (responseToProceed.answer == "n") {
console.log("\nProcess finished by the user! \n");
process.exit(1);
} else {
const responseAfterEnquiry = await prompt({
type: 'input',
name: 'width',
message: 'Inform the width: e.g 200\n'
});
if (responseAfterEnquiry.width) {
imageData.sizeX = responseAfterEnquiry.width;
}
}
}
async function metadataHeigth() {
const responseToProceed = await prompt({
type: 'input',
name: 'answer',
message: 'Metadata height not present do you want to inform it? (Y/N)\n'
});
if (responseToProceed.answer == "n") {
console.log("\nProcess finished by the user! \n");
process.exit(1);
} else {
const responseAfterEnquiry = await prompt({
type: 'input',
name: 'height',
message: 'Inform the height: e.g 400\n'
});
if (responseAfterEnquiry.height) {
imageData.sizeY = responseAfterEnquiry.height;
}
}
}
async function metadataChannels() {
const responseToProceed = await prompt({
type: 'input',
name: 'answer',
message: 'Metadata channels not present do you want to inform it? (Y/N)\n'
});
if (responseToProceed.answer == "n") {
console.log("\nProcess finished by the user! \n");
process.exit(1);
} else {
const responseAfterEnquiry = await prompt({
type: 'input',
name: 'channels',
message: 'Inform the number of channels: e.g 3\n'
});
if (responseAfterEnquiry.channels) {
imageData.nc = responseAfterEnquiry.channels;
}
}
}NOTE: Path of |
|
I think the issue is not with the library itself but with the dependencies required to run C++ executables. I tried using the FROM python:3.12
#FROM python:3.12-slimAnd the command is executing. However, the stdout of the C++ command is not streaming live and it takes approx 15 minutes to complete the execution. 
|
Maybe we can setup a script to add the NFTMarkerCreator.js to the system
Probably it depends on dpi and width, height of the image you are providing for the NFT creations.
I will think about, maybe i can create another repository and add the NFT-Marker-Creator-app as git submodule. I planned to do in the past but never had the time. @ThorstenBux in the past made an app to create NFT markers in a server, but i don't think it is working anymore. The repository is this https://github.com/webarkit/NFT-Creator-WS |
|
I'm trying to build the image but it fails at the end because missed P.S. and maybe |
|
@kalwalt Please check the content of requested files
#!/bin/sh
#set -e
# Accept other commands
exec "$@" |
|
I was able to build the docker image with some modifications, creating a container and running a command to build a nft markers. But i avoided to use python as main command, instead i used the classical command |
|
@anuj9196 tested your mod to |
kalwalt
added a commit
that referenced
this issue
Nov 2, 2024
…th it - read the discussion in issue #5
Merged
kalwalt
added a commit
that referenced
this issue
Nov 3, 2024
…th it - read the discussion in issue #5
|
@anuj9196 did you solved this issue? |
|
@kalwalt Yes, I managed to run it in the docker. Here is how I'm doing it
FROM python:3.12
ENV PYTHONUNBUFFERED=1
# Create a group and user to run our app
ARG APP_USER=myapp
RUN groupadd -r ${APP_USER} && useradd --no-log-init -r -m -g ${APP_USER} ${APP_USER}
# Install packages needed to run your application (not build deps):
RUN set -ex \
&& RUN_DEPS=" \
...
nodejs npm\
" \
&& seq 1 8 | xargs -I{} mkdir -p /usr/share/man/man{} \
&& apt-get update && apt-get install -y --no-install-recommends $RUN_DEPS \
&& rm -rf /var/lib/apt/lists/* \
# Create directories
&& mkdir /app/
# Copy in your requirements file
ADD requirements/ /
WORKDIR /app/
COPY nft-marker-creator /nft-marker-creator
# Install node dependency
RUN npm install --prefix /nft-marker-creator
RUN set -ex \
&& BUILD_DEPS=" \
build-essential \
libpcre3-dev \
libpq-dev \
" \
&& apt-get update && apt-get install -y --no-install-recommends $BUILD_DEPS \
&& pip install --no-cache-dir -r /dev-requirements.txt \
\
&& apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false $BUILD_DEPS \
&& rm -rf /var/lib/apt/lists/*
COPY ./src /app/
# uWSGI will listen on this port
EXPOSE 8000
...
# Change to a non-root user
USER ${APP_USER}:${APP_USER}
ENTRYPOINT ["/scripts/docker/entrypoint.sh"]Installing Then I have created this python module, that accepts the files and downloads it, generates marker and uploads to S3. import os
import tempfile
from urllib.parse import urlparse
import requests
import logging
import subprocess
from django.core.files.base import ContentFile
from django.core.files.storage import default_storage
log = logging.getLogger('app')
def run_command(command):
log.info(f'Running command: {" ".join(command)}')
# Using Popen to capture real-time output
process = subprocess.Popen(
command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
universal_newlines=True # Ensures text output instead of bytes
)
# Reading output in real-time
for stdout_line in iter(process.stdout.readline, ""):
log.info(f"stdout: {stdout_line.strip()}") # Print or log real-time stdout
for stderr_line in iter(process.stderr.readline, ""):
log.info(f"stderr: {stderr_line.strip()}") # Print or log real-time stderr
process.stdout.close()
process.stderr.close()
return_code = process.wait() # Wait for the process to finish
if return_code:
log.info(f"Process finished with code {return_code}")
return return_code
return False
def download_file_to_temp(url):
# Create a temporary file path
temp_dir = tempfile.gettempdir()
file_name = os.path.basename(url)
temp_file_path = os.path.join(temp_dir, file_name)
log.info(f'temp dir: {temp_dir}')
log.info(f'file name: {file_name}')
log.info(f'temp file path: {temp_file_path}')
# Download the file from the URL
try:
log.info(f'downloading file using request: {url}')
response = requests.get(url, stream=True)
response.raise_for_status() # Check for HTTP errors
log.info('Writing response to file')
# Write the file to the temporary path
with open(temp_file_path, 'wb') as temp_file:
for chunk in response.iter_content(chunk_size=8192):
temp_file.write(chunk)
log.info(f'File downloaded to: {temp_file_path}')
except Exception as e:
log.error(f'Error downloading file: {e}')
return None
return temp_file_path
def _upload_directory_to_s3(local_dir, s3__dir_key):
"""
Uploads all files from a local directory to an S3 bucket directory.
:param local_dir: The path to the local directory.
:param s3__dir_key: The directory path within the S3 bucket (optional).
"""
# Traverse through all files in the directory
for root, _, files in os.walk(local_dir):
for file_name in files:
# Construct a full local file path
local_file_path = os.path.join(root, file_name)
# Construct S3 file path (preserving directory structure)
relative_path = os.path.relpath(local_file_path, local_dir)
s3_file_path = os.path.join(s3__dir_key, relative_path).replace("\\", "/") # for Windows compatibility
# Open the file and save it to S3
with open(local_file_path, 'rb') as file_data:
file_content = ContentFile(file_data.read())
default_storage.save(s3_file_path, file_content)
log.info(f"Uploaded {file_name} to S3 at {s3_file_path}")
def get_output_dir():
temp_dir = tempfile.gettempdir()
temp_file_path = os.path.join(temp_dir, 'output/')
return temp_file_path
def create_marker(file_url):
s3_url = file_url
parsed_url = urlparse(s3_url)
# The path after the bucket name, excluding the filename
path_parts = parsed_url.path.strip('/').split('/')
s3_dir_key = '/'.join(path_parts[:-1]) + '/'
file_path = download_file_to_temp(url=s3_url)
output_dir = get_output_dir()
log.info(f'file path is: {file_path}')
log.info(f'output dir is: {output_dir}')
# Check node version
run_command(["node", "--version"])
# Node.js command to run app.js with the -i flag and image path
command = ["node", "/nft-marker-creator/app.js", "-i", file_path, "-o", output_dir]
log.info(f'Executing marker command')
run_command(command)
log.info('Command execution complete')
log.info(f'Listing files of output directory: {output_dir}')
run_command(["ls", "-la", output_dir])
log.info('All execution complete')
log.info(f'Uploading output file to S3 path: {s3_dir_key}')
_upload_directory_to_s3(output_dir, s3_dir_key)usages of python module from nft import create_marker
create_marker('https://bucket.s3.amazonaws.com/path/to/image.png')The python module can be modified to support downloading of maker, or return marker files instead of uploading to s3. |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
I have been trying to run the application to generate markers in a docker container running
Pythonas base image andnode 18as additional dependency. But it stucks onFull
Dockerfileis likeThen running the node application using `
NOTE: I have made some changes to the
src/NFTMarkerCreator.jsfile to allow absolute path to the file and output directoryThe text was updated successfully, but these errors were encountered: