Spaces:
Sleeping
Sleeping
change permission
Browse files- Dockerfile +10 -2
- app.py +76 -30
- download_images.py +38 -9
Dockerfile
CHANGED
|
@@ -10,7 +10,7 @@ RUN apt-get update && \
|
|
| 10 |
|
| 11 |
# Create a non-root user to run the application
|
| 12 |
RUN useradd -m appuser && \
|
| 13 |
-
mkdir -p /home/appuser/app /home/appuser/.cache && \
|
| 14 |
chown -R appuser:appuser /home/appuser
|
| 15 |
|
| 16 |
# Copy requirements first to leverage Docker cache
|
|
@@ -22,7 +22,9 @@ COPY . .
|
|
| 22 |
|
| 23 |
# Create upload directory with proper permissions
|
| 24 |
RUN mkdir -p static/uploads static/metadata && \
|
| 25 |
-
chmod -R 777 static
|
|
|
|
|
|
|
| 26 |
|
| 27 |
# Set environment variables for Hugging Face
|
| 28 |
ENV PYTHONUNBUFFERED=1
|
|
@@ -47,6 +49,12 @@ RUN --mount=type=secret,id=HF_TOKEN,mode=0444,required=true \
|
|
| 47 |
echo "Hugging Face credentials not set, skipping image download."; \
|
| 48 |
fi
|
| 49 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 50 |
# Expose port for Hugging Face Spaces (uses port 7860)
|
| 51 |
EXPOSE 7860
|
| 52 |
|
|
|
|
| 10 |
|
| 11 |
# Create a non-root user to run the application
|
| 12 |
RUN useradd -m appuser && \
|
| 13 |
+
mkdir -p /home/appuser/app /home/appuser/.cache /home/appuser/.image_uploader && \
|
| 14 |
chown -R appuser:appuser /home/appuser
|
| 15 |
|
| 16 |
# Copy requirements first to leverage Docker cache
|
|
|
|
| 22 |
|
| 23 |
# Create upload directory with proper permissions
|
| 24 |
RUN mkdir -p static/uploads static/metadata && \
|
| 25 |
+
chmod -R 777 static && \
|
| 26 |
+
mkdir -p /tmp/.image_uploader && \
|
| 27 |
+
chmod -R 777 /tmp/.image_uploader
|
| 28 |
|
| 29 |
# Set environment variables for Hugging Face
|
| 30 |
ENV PYTHONUNBUFFERED=1
|
|
|
|
| 49 |
echo "Hugging Face credentials not set, skipping image download."; \
|
| 50 |
fi
|
| 51 |
|
| 52 |
+
# Set proper ownership for all files
|
| 53 |
+
RUN chown -R appuser:appuser /app
|
| 54 |
+
|
| 55 |
+
# Switch to non-root user
|
| 56 |
+
USER appuser
|
| 57 |
+
|
| 58 |
# Expose port for Hugging Face Spaces (uses port 7860)
|
| 59 |
EXPOSE 7860
|
| 60 |
|
app.py
CHANGED
|
@@ -39,10 +39,22 @@ METADATA_DIR = Path("static/metadata")
|
|
| 39 |
METADATA_DIR.mkdir(parents=True, exist_ok=True)
|
| 40 |
METADATA_FILE = METADATA_DIR / "image_metadata.json"
|
| 41 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 42 |
# Initialize metadata file if it doesn't exist
|
| 43 |
-
if not METADATA_FILE.exists():
|
| 44 |
-
|
| 45 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 46 |
|
| 47 |
# Mount static directory
|
| 48 |
app.mount("/static", StaticFiles(directory="static"), name="static")
|
|
@@ -138,44 +150,78 @@ def verify_auth(request: Request):
|
|
| 138 |
)
|
| 139 |
return True
|
| 140 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 141 |
def get_image_metadata():
|
| 142 |
"""Get all image metadata including hashtags from local storage and sync with HF if needed."""
|
| 143 |
-
|
| 144 |
-
with open(METADATA_FILE, "r") as f:
|
| 145 |
-
metadata = json.load(f)
|
| 146 |
-
|
| 147 |
-
# In production, sync metadata to Hugging Face if it exists locally but not on HF
|
| 148 |
-
if (
|
| 149 |
-
os.environ.get("ENV", "development") == "production"
|
| 150 |
-
and HF_USERNAME
|
| 151 |
-
and HF_TOKEN
|
| 152 |
-
):
|
| 153 |
-
try:
|
| 154 |
-
# Only upload if there are changes (we'd need to implement a proper change tracking mechanism)
|
| 155 |
-
# For now, we'll upload every time to ensure consistency
|
| 156 |
-
metadata_str = json.dumps(metadata)
|
| 157 |
-
hf_api.upload_file(
|
| 158 |
-
path_or_fileobj=io.BytesIO(metadata_str.encode()),
|
| 159 |
-
path_in_repo=f"{METADATA_PATH}/image_metadata.json",
|
| 160 |
-
repo_id=f"{HF_USERNAME}/{DATASET_REPO}",
|
| 161 |
-
repo_type="dataset",
|
| 162 |
-
token=HF_TOKEN,
|
| 163 |
-
)
|
| 164 |
-
except Exception as e:
|
| 165 |
-
print(f"Error syncing metadata to Hugging Face: {e}")
|
| 166 |
|
| 167 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 168 |
|
| 169 |
# If metadata file doesn't exist locally, create it
|
| 170 |
-
with open(
|
| 171 |
json.dump({}, f)
|
| 172 |
return {}
|
| 173 |
|
| 174 |
def save_image_metadata(metadata):
|
| 175 |
"""Save image metadata to the local JSON file and sync with HF."""
|
|
|
|
|
|
|
| 176 |
# Always save locally first
|
| 177 |
-
|
| 178 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 179 |
|
| 180 |
# In production, also save to Hugging Face
|
| 181 |
if os.environ.get("ENV", "development") == "production" and HF_USERNAME and HF_TOKEN:
|
|
|
|
| 39 |
METADATA_DIR.mkdir(parents=True, exist_ok=True)
|
| 40 |
METADATA_FILE = METADATA_DIR / "image_metadata.json"
|
| 41 |
|
| 42 |
+
# Alternative metadata location with guaranteed write permissions
|
| 43 |
+
HOME_DIR = Path(os.environ.get("HOME", "/tmp"))
|
| 44 |
+
ALT_METADATA_DIR = HOME_DIR / ".image_uploader"
|
| 45 |
+
ALT_METADATA_DIR.mkdir(parents=True, exist_ok=True)
|
| 46 |
+
ALT_METADATA_FILE = ALT_METADATA_DIR / "image_metadata.json"
|
| 47 |
+
|
| 48 |
# Initialize metadata file if it doesn't exist
|
| 49 |
+
if not METADATA_FILE.exists() and not ALT_METADATA_FILE.exists():
|
| 50 |
+
try:
|
| 51 |
+
with open(METADATA_FILE, "w") as f:
|
| 52 |
+
json.dump({}, f)
|
| 53 |
+
print(f"Initialized metadata file at {METADATA_FILE}")
|
| 54 |
+
except PermissionError:
|
| 55 |
+
with open(ALT_METADATA_FILE, "w") as f:
|
| 56 |
+
json.dump({}, f)
|
| 57 |
+
print(f"Initialized metadata file at alternative location: {ALT_METADATA_FILE}")
|
| 58 |
|
| 59 |
# Mount static directory
|
| 60 |
app.mount("/static", StaticFiles(directory="static"), name="static")
|
|
|
|
| 150 |
)
|
| 151 |
return True
|
| 152 |
|
| 153 |
+
|
| 154 |
+
def get_metadata_file():
|
| 155 |
+
"""Get the appropriate metadata file based on write permissions."""
|
| 156 |
+
# Try to write to the primary location
|
| 157 |
+
try:
|
| 158 |
+
if not METADATA_FILE.exists():
|
| 159 |
+
with open(METADATA_FILE, "w") as f:
|
| 160 |
+
json.dump({}, f)
|
| 161 |
+
# Test write permission
|
| 162 |
+
os.access(METADATA_FILE, os.W_OK)
|
| 163 |
+
return METADATA_FILE
|
| 164 |
+
except (PermissionError, OSError):
|
| 165 |
+
print(
|
| 166 |
+
f"Warning: Cannot write to {METADATA_FILE}, using alternative location: {ALT_METADATA_FILE}"
|
| 167 |
+
)
|
| 168 |
+
if not ALT_METADATA_FILE.exists():
|
| 169 |
+
with open(ALT_METADATA_FILE, "w") as f:
|
| 170 |
+
json.dump({}, f)
|
| 171 |
+
return ALT_METADATA_FILE
|
| 172 |
+
|
| 173 |
+
|
| 174 |
def get_image_metadata():
|
| 175 |
"""Get all image metadata including hashtags from local storage and sync with HF if needed."""
|
| 176 |
+
metadata_file = get_metadata_file()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 177 |
|
| 178 |
+
if metadata_file.exists():
|
| 179 |
+
try:
|
| 180 |
+
with open(metadata_file, "r") as f:
|
| 181 |
+
metadata = json.load(f)
|
| 182 |
+
|
| 183 |
+
# In production, sync metadata to Hugging Face if it exists locally but not on HF
|
| 184 |
+
if (
|
| 185 |
+
os.environ.get("ENV", "development") == "production"
|
| 186 |
+
and HF_USERNAME
|
| 187 |
+
and HF_TOKEN
|
| 188 |
+
):
|
| 189 |
+
try:
|
| 190 |
+
# Only upload if there are changes (we'd need to implement a proper change tracking mechanism)
|
| 191 |
+
# For now, we'll upload every time to ensure consistency
|
| 192 |
+
metadata_str = json.dumps(metadata)
|
| 193 |
+
hf_api.upload_file(
|
| 194 |
+
path_or_fileobj=io.BytesIO(metadata_str.encode()),
|
| 195 |
+
path_in_repo=f"{METADATA_PATH}/image_metadata.json",
|
| 196 |
+
repo_id=f"{HF_USERNAME}/{DATASET_REPO}",
|
| 197 |
+
repo_type="dataset",
|
| 198 |
+
token=HF_TOKEN,
|
| 199 |
+
)
|
| 200 |
+
except Exception as e:
|
| 201 |
+
print(f"Error syncing metadata to Hugging Face: {e}")
|
| 202 |
+
|
| 203 |
+
return metadata
|
| 204 |
+
except Exception as e:
|
| 205 |
+
print(f"Error reading metadata file: {e}")
|
| 206 |
+
return {}
|
| 207 |
|
| 208 |
# If metadata file doesn't exist locally, create it
|
| 209 |
+
with open(metadata_file, "w") as f:
|
| 210 |
json.dump({}, f)
|
| 211 |
return {}
|
| 212 |
|
| 213 |
def save_image_metadata(metadata):
|
| 214 |
"""Save image metadata to the local JSON file and sync with HF."""
|
| 215 |
+
metadata_file = get_metadata_file()
|
| 216 |
+
|
| 217 |
# Always save locally first
|
| 218 |
+
try:
|
| 219 |
+
with open(metadata_file, "w") as f:
|
| 220 |
+
json.dump(metadata, f)
|
| 221 |
+
print(f"Metadata saved to {metadata_file}")
|
| 222 |
+
except Exception as e:
|
| 223 |
+
print(f"Error saving metadata locally: {e}")
|
| 224 |
+
# In case of a file error, we'll still try to save to HF
|
| 225 |
|
| 226 |
# In production, also save to Hugging Face
|
| 227 |
if os.environ.get("ENV", "development") == "production" and HF_USERNAME and HF_TOKEN:
|
download_images.py
CHANGED
|
@@ -40,10 +40,34 @@ UPLOAD_DIR = Path("static/uploads")
|
|
| 40 |
METADATA_DIR = Path("static/metadata")
|
| 41 |
METADATA_FILE = METADATA_DIR / "image_metadata.json"
|
| 42 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 43 |
# Create directories if they don't exist
|
| 44 |
UPLOAD_DIR.mkdir(parents=True, exist_ok=True)
|
| 45 |
METADATA_DIR.mkdir(parents=True, exist_ok=True)
|
| 46 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 47 |
# Initialize HfApi
|
| 48 |
hf_api = HfApi(token=HF_TOKEN)
|
| 49 |
|
|
@@ -56,7 +80,7 @@ try:
|
|
| 56 |
# Download metadata first
|
| 57 |
print(f"Downloading metadata from {HF_USERNAME}/{DATASET_REPO}")
|
| 58 |
try:
|
| 59 |
-
|
| 60 |
repo_id=f"{HF_USERNAME}/{DATASET_REPO}",
|
| 61 |
filename=f"{METADATA_PATH}/image_metadata.json",
|
| 62 |
repo_type="dataset",
|
|
@@ -64,21 +88,24 @@ try:
|
|
| 64 |
local_dir=os.path.join(tempfile.gettempdir(), "hf_downloads"),
|
| 65 |
)
|
| 66 |
|
| 67 |
-
print(f"Metadata downloaded to {
|
| 68 |
-
with open(
|
| 69 |
metadata = json.load(f)
|
| 70 |
|
| 71 |
-
# Save metadata locally
|
| 72 |
-
|
|
|
|
| 73 |
json.dump(metadata, f)
|
| 74 |
-
print(f"Metadata saved to {
|
| 75 |
except Exception as e:
|
| 76 |
print(f"Error downloading metadata: {e}")
|
| 77 |
print("Creating empty metadata file")
|
| 78 |
metadata = {}
|
| 79 |
-
# Initialize metadata file
|
| 80 |
-
|
|
|
|
| 81 |
json.dump({}, f)
|
|
|
|
| 82 |
|
| 83 |
# List all files in the dataset
|
| 84 |
print("Listing files in the dataset")
|
|
@@ -123,5 +150,7 @@ try:
|
|
| 123 |
except Exception as e:
|
| 124 |
print(f"Error: {e}")
|
| 125 |
print("Creating empty metadata file")
|
| 126 |
-
|
|
|
|
| 127 |
json.dump({}, f)
|
|
|
|
|
|
| 40 |
METADATA_DIR = Path("static/metadata")
|
| 41 |
METADATA_FILE = METADATA_DIR / "image_metadata.json"
|
| 42 |
|
| 43 |
+
# Alternative metadata location with guaranteed write permissions
|
| 44 |
+
HOME_DIR = Path(os.environ.get("HOME", "/tmp"))
|
| 45 |
+
ALT_METADATA_DIR = HOME_DIR / ".image_uploader"
|
| 46 |
+
ALT_METADATA_DIR.mkdir(parents=True, exist_ok=True)
|
| 47 |
+
ALT_METADATA_FILE = ALT_METADATA_DIR / "image_metadata.json"
|
| 48 |
+
|
| 49 |
# Create directories if they don't exist
|
| 50 |
UPLOAD_DIR.mkdir(parents=True, exist_ok=True)
|
| 51 |
METADATA_DIR.mkdir(parents=True, exist_ok=True)
|
| 52 |
|
| 53 |
+
|
| 54 |
+
# Function to get the appropriate metadata file
|
| 55 |
+
def get_metadata_file():
|
| 56 |
+
# Try to write to the primary location
|
| 57 |
+
try:
|
| 58 |
+
if not METADATA_FILE.exists():
|
| 59 |
+
with open(METADATA_FILE, "w") as f:
|
| 60 |
+
json.dump({}, f)
|
| 61 |
+
# Test write permission
|
| 62 |
+
if os.access(METADATA_FILE, os.W_OK):
|
| 63 |
+
return METADATA_FILE
|
| 64 |
+
raise PermissionError(f"No write permission for {METADATA_FILE}")
|
| 65 |
+
except (PermissionError, OSError) as e:
|
| 66 |
+
print(f"Warning: Cannot use {METADATA_FILE}: {e}")
|
| 67 |
+
print(f"Using alternative location: {ALT_METADATA_FILE}")
|
| 68 |
+
return ALT_METADATA_FILE
|
| 69 |
+
|
| 70 |
+
|
| 71 |
# Initialize HfApi
|
| 72 |
hf_api = HfApi(token=HF_TOKEN)
|
| 73 |
|
|
|
|
| 80 |
# Download metadata first
|
| 81 |
print(f"Downloading metadata from {HF_USERNAME}/{DATASET_REPO}")
|
| 82 |
try:
|
| 83 |
+
metadata_file_path = hf_api.hf_hub_download(
|
| 84 |
repo_id=f"{HF_USERNAME}/{DATASET_REPO}",
|
| 85 |
filename=f"{METADATA_PATH}/image_metadata.json",
|
| 86 |
repo_type="dataset",
|
|
|
|
| 88 |
local_dir=os.path.join(tempfile.gettempdir(), "hf_downloads"),
|
| 89 |
)
|
| 90 |
|
| 91 |
+
print(f"Metadata downloaded to {metadata_file_path}")
|
| 92 |
+
with open(metadata_file_path, "r") as f:
|
| 93 |
metadata = json.load(f)
|
| 94 |
|
| 95 |
+
# Save metadata locally to the appropriate file
|
| 96 |
+
save_path = get_metadata_file()
|
| 97 |
+
with open(save_path, "w") as f:
|
| 98 |
json.dump(metadata, f)
|
| 99 |
+
print(f"Metadata saved to {save_path}")
|
| 100 |
except Exception as e:
|
| 101 |
print(f"Error downloading metadata: {e}")
|
| 102 |
print("Creating empty metadata file")
|
| 103 |
metadata = {}
|
| 104 |
+
# Initialize metadata file
|
| 105 |
+
save_path = get_metadata_file()
|
| 106 |
+
with open(save_path, "w") as f:
|
| 107 |
json.dump({}, f)
|
| 108 |
+
print(f"Created empty metadata file at {save_path}")
|
| 109 |
|
| 110 |
# List all files in the dataset
|
| 111 |
print("Listing files in the dataset")
|
|
|
|
| 150 |
except Exception as e:
|
| 151 |
print(f"Error: {e}")
|
| 152 |
print("Creating empty metadata file")
|
| 153 |
+
save_path = get_metadata_file()
|
| 154 |
+
with open(save_path, "w") as f:
|
| 155 |
json.dump({}, f)
|
| 156 |
+
print(f"Created empty metadata file at {save_path}")
|