I just followed this tutorial : https://github.com/GoogleCloudPlatform/serverless-expeditions/tree/main/cloud-run-django-terraform
I've done :
gcloud builds submit --config cloudbuild-migrate.yaml
terraform apply -var project=$PROJECT_ID
My database was created. Service was pushed online well and I've got a service_url.
When I access it, it shows a 404 error. Even if try to access the /admin page, it just returns a 500 error with no explanation in the log journal (or I didn't find it).
When I try to runserver on localhost with sql_cloud_proxy, Django works perfectly. It's just a Django basic project showing hello on homepage.
I don't get it.
Here's my settings.py file :
import io
import os
from pathlib import Path
import environ
import google.auth
from google.cloud import secretmanager
BASE_DIR = Path(__file__).resolve().parent.parent
env = environ.Env(DEBUG=(bool, False))
env_file = os.path.join(BASE_DIR, ".env")
# Attempt to load the Project ID into the environment, safely failing on error.
try:
_, os.environ["GOOGLE_CLOUD_PROJECT"] = google.auth.default()
except google.auth.exceptions.DefaultCredentialsError:
pass
if os.path.isfile(env_file):
# Use a local secret file, if provided
env.read_env(env_file)
elif os.environ.get("GOOGLE_CLOUD_PROJECT", None):
# Pull secrets from Secret Manager
project_id = os.environ.get("GOOGLE_CLOUD_PROJECT")
client = secretmanager.SecretManagerServiceClient()
settings_name = os.environ.get("SETTINGS_NAME", "django_settings")
name = f"projects/{project_id}/secrets/{settings_name}/versions/latest"
payload = client.access_secret_version(name=name).payload.data.decode("UTF-8")
env.read_env(io.StringIO(payload))
else:
raise Exception("No local .env or GOOGLE_CLOUD_PROJECT detected. No secrets found.")
SECRET_KEY = env("SECRET_KEY")
# DEBUG = env("DEBUG")
DEBUG = True
ALLOWED_HOSTS = ["*"]
INSTALLED_APPS = [
"django.contrib.admin",
"django.contrib.auth",
"django.contrib.contenttypes",
"django.contrib.sessions",
"django.contrib.messages",
"django.contrib.staticfiles",
"gadsapimonitor",
"storages",
]
MIDDLEWARE = [
"django.middleware.security.SecurityMiddleware",
"django.contrib.sessions.middleware.SessionMiddleware",
"django.middleware.common.CommonMiddleware",
"django.middleware.csrf.CsrfViewMiddleware",
"django.contrib.auth.middleware.AuthenticationMiddleware",
"django.contrib.messages.middleware.MessageMiddleware",
"django.middleware.clickjacking.XFrameOptionsMiddleware",
]
ROOT_URLCONF = "gadsapimonitor.urls"
TEMPLATES = [
{
"BACKEND": "django.template.backends.django.DjangoTemplates",
"DIRS": [],
"APP_DIRS": True,
"OPTIONS": {
"context_processors": [
"django.template.context_processors.debug",
"django.template.context_processors.request",
"django.contrib.auth.context_processors.auth",
"django.contrib.messages.context_processors.messages",
],
},
},
]
WSGI_APPLICATION = "gadsapimonitor.wsgi.application"
# Use django-environ to parse DATABASE_URL from secrets
DATABASES = {"default": env.db()}
# If using Cloud SQL Auth Proxy, change the database values accordingly.
if os.environ.get("USE_CLOUD_SQL_AUTH_PROXY"):
DATABASES["default"]["HOST"] = "127.0.0.1"
DATABASES["default"]["PORT"] = 5432
else:
DATABASES["default"]["HOST"] = "127.0.0.1"
DATABASES["default"]["PORT"] = 5432
AUTH_PASSWORD_VALIDATORS = [
{
"NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator",
},
{
"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",
},
{
"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",
},
{
"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",
},
]
LANGUAGE_CODE = "fr-fr"
TIME_ZONE = "UTC"
USE_I18N = True
USE_L10N = True
USE_TZ = True
STATIC_ROOT = "/static/"
STATIC_URL = "/static/"
# Define static storage via django-storages[google]
if env("GS_BUCKET_NAME"):
GS_BUCKET_NAME = env("GS_BUCKET_NAME")
STATICFILES_DIRS = []
DEFAULT_FILE_STORAGE = "storages.backends.gcloud.GoogleCloudStorage"
STATICFILES_STORAGE = "storages.backends.gcloud.GoogleCloudStorage"
GS_DEFAULT_ACL = "publicRead"
DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
My Dockerfile :
# Use an official lightweight Python image.
# https://hub.docker.com/_/python
FROM python:3.9-slim
ENV APP_HOME /app
WORKDIR $APP_HOME
# Install dependencies.
COPY requirements.txt .
RUN pip install -r requirements.txt
# Copy local code to the container image.
COPY . .
# Service must listen to $PORT environment variable.
# This default value facilitates local development.
ENV PORT 8080
# Setting this ensures print statements and log messages
# promptly appear in Cloud Logging.
ENV PYTHONUNBUFFERED TRUE
# Run the web service on container startup. Here we use the gunicorn
# webserver, with one worker process and 8 threads.
# For environments with multiple CPU cores, increase the number of workers
# to be equal to the cores available.
CMD exec gunicorn --bind 0.0.0.0:$PORT --workers 1 --threads 8 --timeout 0 myapp.wsgi:application
I've tried to set DEBUG=True to show something but nothing changed.
Is someone has an idea to fix this issue ?
Many thanks,
EDIT : Log
{
httpRequest: {10}
insertId: "61bd16c6000351727a95b33c"
labels: {
instanceId: "00bf4bf02de1a8b1de1bec9253140c16ac133ee11a6264fd4b1ce063dee5b680b3e567f87e6e1ec1a40e92314275ca0dac93e25dc02cded8677918"
}
logName: "projects/gaxxxxxxxtor-xxxx420/logs/run.googleapis.com%2Frequests"
receiveTimestamp: "2021-12-17T23:01:26.220970335Z"
resource: {
labels: {
configuration_name: "gaxxxxxxxtor-xxxx420"
location: "europe-west4"
project_id: "gaxxxxxxxtor-xxxx420"
revision_name: "gaxxxxxxxtor-xxxx420-kw9ld"
service_name: "gaxxxxxxxtor-xxxx420"
}
type: "cloud_run_revision"
}
severity: "ERROR"
timestamp: "2021-12-17T23:01:26.217458Z"
trace: "projects/gaxxxxxxxtor-xxxx420/traces/3e655015a967bfe8903db0ef5d1cfc7f"
}
Related
My Django Application (Largely REST Framework based) is currently producing URLs on the admin page that don't resolve. The expected result is that the Django Admin's login prompt submits the form with a POST to /admin/login. The resultant URL passed by as the form submission URL by Django is /$default/admin/login and that returns a 404 with the even more obtuse /$default/$default/admin/login/.
I'm presuming I have some sort of misconfiguration in either my Django configuration or serverless.yml.
As per the following serverless.yml I'm using API Gateway V2, Django through WSGI, and Lambda functions.
service: api
app: api
org: myapp
frameworkVersion: '3'
provider:
name: aws
runtime: python3.8
functions:
serve:
handler: wsgi_handler.handler
timeout: 20
environment:
DB_NAME: ${param:db_name}
DB_PASSWORD: ${param:db_password}
DB_USER: ${param:db_user}
DB_PORT: ${param:db_port}
DB_HOST: ${param:db_host}
events:
- httpApi: "*"
migration:
handler: migrate.handler
timeout: 60
environment:
DB_NAME: ${param:db_name}
DB_PASSWORD: ${param:db_password}
DB_USER: ${param:db_user}
DB_PORT: ${param:db_port}
DB_HOST: ${param:db_host}
custom:
wsgi:
app: myapp.wsgi.application
plugins:
- serverless-python-requirements
- serverless-wsgi
My URLs are pretty standard:
from django.contrib import admin
from django.urls import path, include
from rest_framework.schemas import get_schema_view
schema_view = get_schema_view(
title="MyApp",
description="MyApp Universal API",
version="1.0.0",
)
urlpatterns = [
path("admin/", admin.site.urls),
path("user/", include("myapp.core.urls"), name="user"),
path("openapi", schema_view, name="openapi-schema"),
]
My configuration is even more standard:
import os
from pathlib import Path
from dotenv import load_dotenv
load_dotenv()
# Build paths inside the project like this: BASE_DIR / 'subdir'.
BASE_DIR = Path(__file__).resolve().parent.parent
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/4.1/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = "not for you :)"
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
ALLOWED_HOSTS = ['*']
if 'CODESPACE_NAME' in os.environ:
codespace_name = os.getenv("CODESPACE_NAME")
codespace_domain = os.getenv("GITHUB_CODESPACES_PORT_FORWARDING_DOMAIN")
CSRF_TRUSTED_ORIGINS = [f'https://{codespace_name}-8000.{codespace_domain}']
ROOT_URLCONF = "myapp.urls"
# Application definition
INSTALLED_APPS = [
"django.contrib.admin",
"django.contrib.auth",
"django.contrib.contenttypes",
"django.contrib.sessions",
"django.contrib.messages",
"django.contrib.staticfiles",
'rest_framework',
"myapp.core",
]
MIDDLEWARE = [
"django.middleware.security.SecurityMiddleware",
"django.contrib.sessions.middleware.SessionMiddleware",
"django.middleware.common.CommonMiddleware",
"django.middleware.csrf.CsrfViewMiddleware",
"django.contrib.auth.middleware.AuthenticationMiddleware",
"django.contrib.messages.middleware.MessageMiddleware",
"django.middleware.clickjacking.XFrameOptionsMiddleware",
]
X_FRAME_OPTIONS = "ALLOW-FROM preview.app.github.dev"
ROOT_URLCONF = "myapp.urls"
TEMPLATES = [
{
"BACKEND": "django.template.backends.django.DjangoTemplates",
"DIRS": [BASE_DIR / "myapp" / "templates"],
"APP_DIRS": True,
"OPTIONS": {
"context_processors": [
"django.template.context_processors.debug",
"django.template.context_processors.request",
"django.contrib.auth.context_processors.auth",
"django.contrib.messages.context_processors.messages",
],
},
},
]
WSGI_APPLICATION = "myapp.wsgi.application"
STATIC_ROOT = os.path.join(BASE_DIR, 'staticfiles')
# Database
# https://docs.djangoproject.com/en/4.1/ref/settings/#databases
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.postgresql',
'NAME': os.environ.get('DB_NAME', 'neondb'),
'USER': os.environ.get('DB_USER', 'postgres'),
'PASSWORD': os.environ.get('DB_PASSWORD'),
'HOST': os.environ.get('DB_HOST', 'localhost'),
'PORT': os.environ.get('DB_PORT', 5432),
}
}
# Password validation
# https://docs.djangoproject.com/en/4.1/ref/settings/#auth-password-validators
AUTH_PASSWORD_VALIDATORS = [
{
"NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator",
},
{
"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",
},
{
"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",
},
{
"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",
},
]
# Internationalization
# https://docs.djangoproject.com/en/4.1/topics/i18n/
LANGUAGE_CODE = "en-us"
TIME_ZONE = "UTC"
USE_I18N = True
USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/4.1/howto/static-files/
STATICFILES_DIRS = [
BASE_DIR / "myapp" / "static",
]
STATIC_URL = "static/"
# Default primary key field type
# https://docs.djangoproject.com/en/4.1/ref/settings/#default-auto-field
DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
STATICFILES_STORAGE = 'storages.backends.s3boto3.S3StaticStorage'
AWS_ACCESS_KEY_ID = os.environ.get('AWS_ACCESS_KEY_ID')
AWS_SECRET_ACCESS_KEY = os.environ.get('AWS_SECRET_ACCESS_KEY')
AWS_STORAGE_BUCKET_NAME = "myapp-django-static"
AUTH_USER_MODEL = 'core.User'
REST_FRAMEWORK = {
'DEFAULT_RENDERER_CLASSES': (
'rest_framework.renderers.JSONRenderer',
),
'DEFAULT_PERMISSION_CLASSES': (
'rest_framework.permissions.IsAuthenticatedOrReadOnly',
)
}
As for the resultant error message:
Any help (or ideas) would be greatly appreciated!
I've tried modifying the path structure of the serverless.yml and have been trawling through the Django source code for any hints to no avail. Naturally I'd just like Django admin to work. As far as the rest of the app, it works fine as the API itself isn't self referential. Django just isn't returning the correct path.
To put it in brief, Django thinks that all of my routes are prefixed by /$default/ they are not. I'm looking for either a solution to force the path to sent by Django to be / or a fix for my Serverless configuration to mitigate this issue.
I managed to resolve this with a rather niché Django settings option: FORCE_SCRIPT_NAME
While this doesn't explain why it is resolving the path to /$default/ it does mitigate this issue.
If you're using this for a different use case than mine and your path is in a subdirectory (e.g the opposite issue to mine) then you would add your path in FORCE_SCRIPT_NAME instead.
See the link I've provided for more information.
Add the following to your app's settings.py, or any file you're using as your DJANGO_SETTINGS_MODULE.
# Force Django to resolve the URL to the root of the site
# This is required for API Gateway <- WSGI -> Django path resolution to work properly.
FORCE_SCRIPT_NAME = ""
I have built a server with Django and I am receiving a 400 Bad Request within Postman when I check the POST method. However, it is displaying the JSON data within the Postman as well.
Originally, I thought it was a frontend issue, because my console log was stating
AxiosError {message: 'Request failed with status code 400', name: 'AxiosError', code: 'ERR_BAD_REQUEST', config: {…}, request: XMLHttpRequest, …}
As I stated before, the error is showing in Postman so I'm assuming it's actually a server issue.
Im real "finicky" when it comes to file colors, and if it is any color than the default color, I feel like there is errors within the file. Could these orange/beige files contain the problem? If I click on one of those files Im met with:
The file is not displayed in the editor because it is either binary or uses an unsupported text encoding.
Because I am new to django, I don't want to do something that's going to create an even bigger problem than what I already have.
Below will be my settings.py
"""
Django settings for django_rest_api project.
Generated by 'django-admin startproject' using Django 4.1.4.
For more information on this file, see
https://docs.djangoproject.com/en/4.1/topics/settings/
For the full list of settings and their values, see
https://docs.djangoproject.com/en/4.1/ref/settings/
"""
from pathlib import Path
# Build paths inside the project like this: BASE_DIR / 'subdir'.
BASE_DIR = Path(__file__).resolve().parent.parent
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/4.1/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = "django-insecure-2r=5^!2(nj*^*m*sa_39h(xldjpau&$wmn&pc5=^frws#w&25="
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
ALLOWED_HOSTS = []
# Application definition
INSTALLED_APPS = [
"corsheaders",
"rest_framework",
"todo_api",
"django.contrib.admin",
"django.contrib.auth",
"django.contrib.contenttypes",
"django.contrib.sessions",
"django.contrib.messages",
"django.contrib.staticfiles",
]
MIDDLEWARE = [
'corsheaders.middleware.CorsMiddleware',
"django.middleware.security.SecurityMiddleware",
"django.contrib.sessions.middleware.SessionMiddleware",
"django.middleware.common.CommonMiddleware",
"django.middleware.csrf.CsrfViewMiddleware",
"django.contrib.auth.middleware.AuthenticationMiddleware",
"django.contrib.messages.middleware.MessageMiddleware",
"django.middleware.clickjacking.XFrameOptionsMiddleware",
]
CORS_ALLOW_ALL_ORIGINS = True
ROOT_URLCONF = "django_rest_api.urls"
TEMPLATES = [
{
"BACKEND": "django.template.backends.django.DjangoTemplates",
"DIRS": [],
"APP_DIRS": True,
"OPTIONS": {
"context_processors": [
"django.template.context_processors.debug",
"django.template.context_processors.request",
"django.contrib.auth.context_processors.auth",
"django.contrib.messages.context_processors.messages",
],
},
},
]
WSGI_APPLICATION = "django_rest_api.wsgi.application"
# Database
# https://docs.djangoproject.com/en/4.1/ref/settings/#databases
DATABASES = {
"default": {
"ENGINE": "django.db.backends.postgresql",
"NAME": "todo_api",
"USER": "",
"PASSWORD" : '',
"HOST": "localhost"
}
}
# Password validation
# https://docs.djangoproject.com/en/4.1/ref/settings/#auth-password-validators
AUTH_PASSWORD_VALIDATORS = [
{
"NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator",
},
{
"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",
},
{
"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",
},
{
"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",
},
]
# Internationalization
# https://docs.djangoproject.com/en/4.1/topics/i18n/
LANGUAGE_CODE = "en-us"
TIME_ZONE = "UTC"
USE_I18N = True
USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/4.1/howto/static-files/
STATIC_URL = "static/"
# Default primary key field type
# https://docs.djangoproject.com/en/4.1/ref/settings/#default-auto-field
DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
Below is my views.py
from django.shortcuts import render
from rest_framework import generics
from .serializer import TodoSerializer
from .models import Todo
# Create your views here.
class TodoList(generics.ListCreateAPIView):
queryset = Todo.objects.all().order_by('id')
serializer_class = TodoSerializer
class TodoDetail(generics.RetrieveUpdateDestroyAPIView):
queryset = Todo.objects.all().order_by('id')
serializer_class = TodoSerializer
Im not exactly sure what information everyone needs to provide a proper solution for this, but hopefully I have provided more than enough.
I think it is worth noting that I am using postgreql and my database is working properly, and I can access Django admin and manually enter data from there as well.
When running locally my files are correct but on production it seems that none of the changes are through. I might be forgetting to do something to make the production files the same as local. Changed inbox.html to another file name.
TemplateDoesNotExist at /
inbox.html
Request Method: GET
Request URL: https://url.com/
Django Version: 3.2.9
Exception Type: TemplateDoesNotExist
Exception Value:
inbox.html
Exception Location: /app/.heroku/python/lib/python3.9/site-packages/django/template/backends/django.py, line 84, in reraise
Python Executable: /app/.heroku/python/bin/python
Python Version: 3.9.6
Python Path:
['/app/.heroku/python/bin',
'/app',
'/app/.heroku/python/lib/python39.zip',
'/app/.heroku/python/lib/python3.9',
'/app/.heroku/python/lib/python3.9/lib-dynload',
'/app/.heroku/python/lib/python3.9/site-packages']
Server time: Tue, 01 Feb 2022 16:02:49 -0800
settings.py
"""
Django settings for portfolio project.
Generated by 'django-admin startproject' using Django 3.2.9.
For more information on this file, see
https://docs.djangoproject.com/en/3.2/topics/settings/
For the full list of settings and their values, see
https://docs.djangoproject.com/en/3.2/ref/settings/
"""
from pathlib import Path
# Build paths inside the project like this: BASE_DIR / 'subdir'.
BASE_DIR = Path(__file__).resolve().parent.parent
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
import os
#Gets rid of
from decouple import config
SECRET_KEY = config("SECRET_KEY")
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
#DEBUG = config('DJANGO_DEBUG',default=True, cast=bool)
ALLOWED_HOSTS = ["url.com",'127.0.0.1','localhost']
# Application definition
INSTALLED_APPS = [
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
'django.contrib.sites',
'social_django',
'formtools',
'phonenumber_field',
'pages.apps.PagesConfig',
'storages',
'django_cleanup.apps.CleanupConfig',
]
MIDDLEWARE = [
'django.middleware.security.SecurityMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django_session_timeout.middleware.SessionTimeoutMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
'social_django.middleware.SocialAuthExceptionMiddleware', #new
]
ROOT_URLCONF = 'portfolio.urls'
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [os.path.join(BASE_DIR, 'templates')],
'APP_DIRS': True,
'OPTIONS': {
'context_processors': [
'django.template.context_processors.debug',
'django.template.context_processors.request',
'django.contrib.auth.context_processors.auth',
'django.contrib.messages.context_processors.messages',
'social_django.context_processors.backends', # <--
'social_django.context_processors.login_redirect', # <--
],
},
},
]
WSGI_APPLICATION = 'portfolio.wsgi.application'
# Password validation
# https://docs.djangoproject.com/en/3.2/ref/settings/#auth-password-validators
AUTH_PASSWORD_VALIDATORS = [
{
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
},
]
# Internationalization
# https://docs.djangoproject.com/en/3.2/topics/i18n/
# Swapped to Canada
LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'Canada/Pacific'
USE_I18N = True
USE_L10N = True
USE_TZ = False
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/3.2/howto/static-files/
# All of this is in my console.aws.amazon to configure aws s3 static files
# If I am in prod DEBUG==false
# IAM Management Console
if DEBUG:
AWS_ACCESS_KEY_ID = config('AWS_ACCESS_KEY_ID')
AWS_SECRET_ACCESS_KEY = config('AWS_SECRET_ACCESS_KEY')
# Amazon S3 Buckets
AWS_STORAGE_BUCKET_NAME = config('AWS_STORAGE_BUCKET_NAME')
AWS_S3_CUSTOM_DOMAIN = '%s.s3.amazonaws.com' % AWS_STORAGE_BUCKET_NAME
AWS_S3_OBJECT_PARAMETERS = {
'CacheControl': 'max-age=86400',
}
AWS_DEFAULT_ACL = None
AWS_S3_SIGNATURE_VERSION = 's3v4'
AWS_S3_REGION_NAME = 'us-east-2'
AWS_STATIC_LOCATION = 'static'
STATIC_URL = "https://%s/%s/" % (AWS_S3_CUSTOM_DOMAIN, AWS_STATIC_LOCATION)
STATICFILES_STORAGE = 'portfolio.storage_backends.StaticStorage'
MEDIAFILES_LOCATION = 'media'
MEDIA_URL = "https://%s/%s/" % (AWS_S3_CUSTOM_DOMAIN, MEDIAFILES_LOCATION )
DEFAULT_FILE_STORAGE = 'portfolio.storage_backends.MediaStorage'
else:
STATIC_ROOT = os.path.join(BASE_DIR, 'static')
STATIC_URL= '/static/'
MEDIA_ROOT = os.path.join(BASE_DIR, 'media')
MEDIA_URL = '/media/'
STATICFILES_DIRS = [
os.path.join(BASE_DIR, 'pages/static'),
]
# Fixes Found another file with the destination path
STATICFILES_FINDERS = (
'django.contrib.staticfiles.finders.FileSystemFinder',
#'django.contrib.staticfiles.finders.AppDirectoriesFinder', #causes verbose duplicate notifications in django 1.9
)
# Default primary key field type
# https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field
DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField'
#Adding social logins
AUTHENTICATION_BACKENDS = ['social_core.backends.google.GoogleOAuth2','django.contrib.auth.backends.AllowAllUsersModelBackend',]
# Database
# https://docs.djangoproject.com/en/3.2/ref/settings/#databases
# Extends default user with additional fields
AUTH_USER_MODEL = 'pages.Profile'
SOCIAL_AUTH_USER_MODEL = 'pages.Profile'
# social auth configs for google
SOCIAL_AUTH_GOOGLE_OAUTH2_KEY = config('GOOGLE_OAUTH2_KEY')
SOCIAL_AUTH_GOOGLE_OAUTH2_SECRET = config('GOOGLE_OAUTH2_SECRET')
SOCIAL_AUTH_GOOGLE_OAUTH2_SCOPE = ['https://www.googleapis.com/auth/calendar']
SOCIAL_AUTH_JSONFIELD_ENABLED = True
SOCIAL_AUTH_GOOGLE_OAUTH2_AUTH_EXTRA_ARGUMENTS = {'access_type': 'offline',}
SOCIAL_AUTH_PIPELINE = (
'social_core.pipeline.social_auth.social_details',
'social_core.pipeline.social_auth.social_uid',
'social_core.pipeline.social_auth.auth_allowed',
'social_core.pipeline.social_auth.social_user',
'social_core.pipeline.user.get_username',
'social_core.pipeline.social_auth.associate_by_email', # <--- enable this one
'social_core.pipeline.user.create_user',
'social_core.pipeline.social_auth.associate_user',
'social_core.pipeline.social_auth.load_extra_data',
'social_core.pipeline.user.user_details',
)
# Todo:Add a postgresql for prod settings using Amazon RDS Management Console
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.sqlite3',
'NAME': BASE_DIR / 'db.sqlite3',
}
}
import dj_database_url
db_from_env = dj_database_url.config(conn_max_age=600)
DATABASES['default'].update(db_from_env)
LOGIN_URL = 'login'
LOGIN_REDIRECT_URL = 'home'
LOGOUT_REDIRECT_URL = 'home'
SESSION_SERIALIZER = 'django.contrib.sessions.serializers.PickleSerializer'
SESSION_ENGINE = (
'django.contrib.sessions.backends.cache'
)
"""
SESSION_COOKIE_AGE = 60 # change expired session
SESSION_EXPIRE_AT_BROWSER_CLOSE = True
SESSION_SAVE_EVERY_REQUEST = True
"""
import django_heroku
django_heroku.settings(locals(), staticfiles=False)
I changed Inbox.html to another file name but running heroku bash seems like nothing changes.
Running bash on ⬢ arundeepchohan... up, run.9168 (Free)
~ $ ls
google-credentials.json manage.py pages portfolio Procfile Procfile.windows requirements.txt runtime.txt
~ $ cd pages
~/pages $ ls
admin.py calendar.py googleCalendarAPI.py migrations __pycache__ templates urls.py
apps.py forms.py __init__.py models.py static tests.py views.py
~/pages $ cd templates
~/pages/templates $ ls
adminControls.html dashboard.html editProfileModal.html Inbox.html sendMessageForm.html
base.html documents.html googleCalendar.html pickUserType.html sidebar.html
bookAppointment.html editProfileForm.html home.html registration
~/pages/templates $ exit
exit
So I added it to Github. Then I used the automatic deploy function from Heroku to connect it to that repository. Which did allow my heroku run bash to pick up the newly generated files. The only issues now are production database values are now not migrated properly.
I'm following this tutorial
to deploy Django with Google Cloud. At the timestamp (13:39) he goes into settings.py in the Django project in VS Code and swaps some placeholders out for his own Google Cloud credentials. My settings.py however looks very different. The only thing in block caps that looks as if it might be a placeholder is "GOOGLE_CLOUD_PROJECT" and "SETTINGS_NAME". This is the part of the installation I can't get past because at the next step when I try to execute
python manage.py makemigrations
I get sent here:
else:
raise Exception("No local .env or GOOGLE_CLOUD_PROJECT detected. No secrets found.")
Hoping someone can glance over my settings.py and identify what I'm missing why that if block is going to Exception.
One thing to note: The top imports environ and google.cloud are not resolved. idk if that's got anything to do with it. This is how the file came out the can. I don't want to mess with it in case tshtf :D
settings.py
import io
import os
import environ
from google.cloud import secretmanager
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# [START gaestd_py_django_secret_config]
env = environ.Env(DEBUG=(bool, False))
env_file = os.path.join(BASE_DIR, ".env")
if os.path.isfile(env_file):
# Use a local secret file, if provided
env.read_env(env_file)
# [START_EXCLUDE]
elif os.getenv("TRAMPOLINE_CI", None):
# Create local settings if running with CI, for unit testing
placeholder = (
f"SECRET_KEY=a\n"
f"DATABASE_URL=sqlite://{os.path.join(BASE_DIR, 'db.sqlite3')}"
)
env.read_env(io.StringIO(placeholder))
# [END_EXCLUDE]
elif os.environ.get("GOOGLE_CLOUD_PROJECT", None):
# Pull secrets from Secret Manager
project_id = os.environ.get("GOOGLE_CLOUD_PROJECT")
client = secretmanager.SecretManagerServiceClient()
settings_name = os.environ.get("SETTINGS_NAME", "django_settings")
name = f"projects/{project_id}/secrets/{settings_name}/versions/latest"
payload = client.access_secret_version(name=name).payload.data.decode("UTF-8")
env.read_env(io.StringIO(payload))
else:
raise Exception("No local .env or GOOGLE_CLOUD_PROJECT detected. No secrets found.")
# [END gaestd_py_django_secret_config]
SECRET_KEY = env("SECRET_KEY")
# SECURITY WARNING: don't run with debug turned on in production!
# Change this to "False" when you are ready for production
DEBUG = True
# SECURITY WARNING: App Engine's security features ensure that it is safe to
# have ALLOWED_HOSTS = ['*'] when the app is deployed. If you deploy a Django
# app not on App Engine, make sure to set an appropriate host here.
ALLOWED_HOSTS = ["*"]
# Application definition
INSTALLED_APPS = [
"polls.apps.PollsConfig",
"django.contrib.admin",
"django.contrib.auth",
"django.contrib.contenttypes",
"django.contrib.sessions",
"django.contrib.messages",
"django.contrib.staticfiles",
]
MIDDLEWARE = [
"django.middleware.security.SecurityMiddleware",
"django.contrib.sessions.middleware.SessionMiddleware",
"django.middleware.common.CommonMiddleware",
"django.middleware.csrf.CsrfViewMiddleware",
"django.contrib.auth.middleware.AuthenticationMiddleware",
"django.contrib.messages.middleware.MessageMiddleware",
"django.middleware.clickjacking.XFrameOptionsMiddleware",
]
ROOT_URLCONF = "mysite.urls"
TEMPLATES = [
{
"BACKEND": "django.template.backends.django.DjangoTemplates",
"DIRS": [],
"APP_DIRS": True,
"OPTIONS": {
"context_processors": [
"django.template.context_processors.debug",
"django.template.context_processors.request",
"django.contrib.auth.context_processors.auth",
"django.contrib.messages.context_processors.messages",
],
},
},
]
WSGI_APPLICATION = "mysite.wsgi.application"
# Database
# [START db_setup]
# [START gaestd_py_django_database_config]
# Use django-environ to parse the connection string
DATABASES = {"default": env.db()}
# If the flag as been set, configure to use proxy
if os.getenv("USE_CLOUD_SQL_AUTH_PROXY", None):
DATABASES["default"]["HOST"] = "127.0.0.1"
DATABASES["default"]["PORT"] = 5432
# [END gaestd_py_django_database_config]
# [END db_setup]
# Use a in-memory sqlite3 database when testing in CI systems
# TODO(glasnt) CHECK IF THIS IS REQUIRED because we're setting a val above
if os.getenv("TRAMPOLINE_CI", None):
DATABASES = {
"default": {
"ENGINE": "django.db.backends.sqlite3",
"NAME": os.path.join(BASE_DIR, "db.sqlite3"),
}
}
# Password validation
AUTH_PASSWORD_VALIDATORS = [
{
"NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", # noqa: 501
},
{
"NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", # noqa: 501
},
{
"NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", # noqa: 501
},
{
"NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", # noqa: 501
},
]
# Internationalization
LANGUAGE_CODE = "en-us"
TIME_ZONE = "UTC"
USE_I18N = True
USE_L10N = True
USE_TZ = True
# Static files (CSS, JavaScript, Images)
STATIC_ROOT = "static"
STATIC_URL = "/static/"
STATICFILES_DIRS = []
# Default primary key field type
# https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field
DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField"
Django wants you to provide a SECRET_KEY. There are a couple of ways to provide such key, but one way is to create an .env file containing the SECRET_KEY in the same folder where the manage.py file is located.
A SECRET_KEY can be generated using the following code snippet (run it wherever you find convenient, but don't add the resulting key string to your settings.py file):
from django.core.management.utils import get_random_secret_key
get_random_secret_key()
Take the resulting output (it's a 50 character long string) and add it to the .env file with the following format:
SECRET_KEY="the-50-character-string-here"
You might potentially also have to replace BASE_DIR in the settings.py file with the following in order to properly load the .env file (note how the first row is out-commented).
#BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
BASE_DIR = os.getcwd()
You might run into more problems further down the road related to your database setup. Check out this and this SO question.
You can also consider checking out the Guide which is what the tutorial you're referring to is following.
I want to use cronjob to run a script, which is to fetch data from news api and feed it into postegres which is located in other container.
so the simplified architecture is
app(in container) -> postegres(in container)
the cronjob script is inside app, and it will fetch data and then send to postegres.
in my crontab is
* * * * * cd /tourMamaRoot/tourMama/cronjob && fetch_news.py >> /var/log/cron.log 2>&1
i can run it successfully by manually run the script, but when i put it in crontab , it shows the error.
File "/usr/local/lib/python3.6/dist-packages/django/db/backends/base/base.py", line 195, in connect
self.connection = self.get_new_connection(conn_params)
File "/usr/local/lib/python3.6/dist-packages/django/db/backends/postgresql/base.py", line 178, in get_new_connection
connection = Database.connect(**conn_params)
File "/usr/local/lib/python3.6/dist-packages/psycopg2/__init__.py", line 126, in connect
conn = _connect(dsn, connection_factory=connection_factory, **kwasync)
django.db.utils.OperationalError: could not connect to server: No such file or directory
Is the server running locally and accepting
connections on Unix domain socket "/var/run/postgresql/.s.PGSQL.5432"?
seems it only look for database locally if use crontab, how can i set it to put data into other container like i manually run the script?
Info:
my docker container for app is Ubuntu version 18.04 , and the following is my docker file for app
FROM ubuntu:18.04
MAINTAINER Eson
ENV PYTHONUNBUFFERED 1
ENV DEBIAN_FRONTEND=noninteractive
EXPOSE 8000
# Setup directory structure
RUN mkdir /tourMamaRoot
WORKDIR /tourMamaRoot/tourMama/
COPY tourMama/requirements/base.txt /tourMamaRoot/base.txt
COPY tourMama/requirements/dev.txt /tourMamaRoot/requirements.txt
# install Python 3
RUN apt-get update && apt-get install -y \
software-properties-common
RUN add-apt-repository ppa:deadsnakes/ppa
RUN apt-get update && apt-get install -y \
python3.7 \
python3-pip
RUN python3.7 -m pip install pip
RUN apt-get update && apt-get install -y \
python3-distutils \
python3-setuptools
# install Postgresql
RUN apt-get -y install wget ca-certificates
RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
RUN sh -c echo deb http://apt.postgresql.org/pub/repos/apt/ `lsb_release -cs`-pgdg main >> /etc/apt/sources.list.d/pgdg.list
RUN apt-get update
RUN apt-get install -y postgresql postgresql-contrib
# Install some dep
RUN apt-get install net-tools
RUN apt-get install -y libpq-dev python-dev
RUN pip3 install -r /tourMamaRoot/requirements.txt
# Copy application
COPY ./tourMama/ /tourMamaRoot/tourMama/
docker compose file:
version: '3'
services:
app:
build:
# current directory
# if for dev, need to have Dockerfile.dev in folder
dockerfile: docker/dev/Dockerfile
context: .
ports:
#host to image
- "8000:8000"
volumes:
# map directory to image, which means if something changed in
# current directory, it will automatically reflect on image,
# don't need to restart docker to get the changes into effect
- ./tourMama:/tourMamaRoot/tourMama
command: >
sh -c "python3 manage.py wait_for_db &&
python3 manage.py makemigrations &&
python3 manage.py migrate &&
python3 manage.py runserver 0.0.0.0:8000 &&
sh initial_all.sh"
environment:
- DB_HOST=db
- DB_NAME=app
- DB_USER=postgres
- DB_PASS=supersecretpassword
depends_on:
- db
- redis
db:
image: postgres:11-alpine
ports:
#host to image
- "5432:5432"
environment:
- POSTGRES_DB=app
- POSTGRES_USER=postgres
- POSTGRES_PASSWORD=supersecretpassword
redis:
image: redis:5.0.5-alpine
ports:
#host to image
- "6379:6379"
# command: ["redis-server", "--appendonly", "yes"]
# hostname: redis
# networks:
# - redis-net
# volumes:
# - redis-data:/data
and my cronjob script is:
import os
import sys
import django
from django.db import IntegrityError
from newsapi.newsapi_client import NewsApiClient
sys.path.append("../")
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "tourMama.settings")
django.setup()
from news.models import News
from tourMama_app.models import Category
from config.script import categorization_loader
load_category = categorization_loader.load_category_data("catagorization.yml")
categories = list(load_category.keys())
countries = ["us", "gb"]
# Init
newsapi = NewsApiClient(api_key='secret')
for category in categories:
for country in countries:
category_lower = category.lower()
category_obj = Category.objects.filter(
category=category,
).get()
top_headlines = newsapi.get_top_headlines(q='',
# sources=object'bbc-news,the-verge',
category=category_lower,
language='en',
page_size=100,
country=country
)
for article in top_headlines.get("articles"):
try:
News.objects.create(
source=article["source"].get("name") if article["source"] else None,
title=article.get("title"),
author=article.get("author"),
description=article.get("description"),
url=article.get("url"),
urlToImage=article.get("urlToImage"),
published_at=article.get("publishedAt"),
content=article.get("content"),
category=category_obj
)
except IntegrityError:
print("data already exist")
else:
print("data insert successfully")
and if needed, my django setting file is as following:
"""
Django settings for tourMama project.
Generated by 'django-admin startproject' using Django 2.2.1.
For more information on this file, see
https://docs.djangoproject.com/en/2.2/topics/settings/
For the full list of settings and their values, see
https://docs.djangoproject.com/en/2.2/ref/settings/
"""
import os
# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
TEMPLATE_DIR = os.path.join(BASE_DIR,"templates")
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/2.2/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = 'd084cm20*x*&s&w)vq+7*teea540yny+fyi^dh57nxiff&a#25'
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
COMPRESS_ENABLED = False
COMPRESS_CSS_HASHING_METHOD = 'content'
COMPRESS_FILTERS = {
'css':[
'compressor.filters.css_default.CssAbsoluteFilter',
'compressor.filters.cssmin.rCSSMinFilter',
],
'js':[
'compressor.filters.jsmin.JSMinFilter',
]
}
HTML_MINIFY = False
KEEP_COMMENTS_ON_MINIFYING = False
ALLOWED_HOSTS = ['0.0.0.0', "127.0.0.1"]
# Application definition
INSTALLED_APPS = [
'django.contrib.admin',
'django.contrib.auth',
'django.contrib.contenttypes',
'django.contrib.sessions',
'django.contrib.messages',
'django.contrib.staticfiles',
'channels',
'bootstrap3',
'tourMama_app',
'account',
'posts',
'group',
'news',
'statistics',
'compressor',
]
AUTH_USER_MODEL = "account.UserProfile"
MIDDLEWARE = [
'django.middleware.gzip.GZipMiddleware',
'htmlmin.middleware.HtmlMinifyMiddleware',
'htmlmin.middleware.MarkRequestMiddleware',
'django.middleware.security.SecurityMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
]
ROOT_URLCONF = 'tourMama.urls'
TEMPLATES = [
{
'BACKEND': 'django.template.backends.django.DjangoTemplates',
'DIRS': [TEMPLATE_DIR,],
'APP_DIRS': True,
'OPTIONS': {
'context_processors': [
'django.template.context_processors.debug',
'django.template.context_processors.request',
'django.contrib.auth.context_processors.auth',
'django.contrib.messages.context_processors.messages' ,
],
},
},
]
WSGI_APPLICATION = 'tourMama.wsgi.application'
ASGI_APPLICATION = 'tourMama.routing.application'
# https://stackoverflow.com/questions/56480472/cannot-connect-to-redis-container-from-app-container/56480746#56480746
CHANNEL_LAYERS = {
'default': {
'BACKEND': 'channels_redis.core.RedisChannelLayer',
'CONFIG': {
"hosts": [('redis', 6379)],
},
},
}
# Database
# https://docs.djangoproject.com/en/2.2/ref/settings/#databases
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.postgresql',
'HOST': os.environ.get('DB_HOST'),
'NAME': os.environ.get('DB_NAME'),
'USER': os.environ.get('DB_USER'),
'PASSWORD': os.environ.get('DB_PASS')
}
}
# Password validation
# https://docs.djangoproject.com/en/2.2/ref/settings/#auth-password-validators
AUTH_PASSWORD_VALIDATORS = [
{
'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',
},
{
'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',
},
]
STATICFILES_FINDERS = (
'django.contrib.staticfiles.finders.FileSystemFinder',
'django.contrib.staticfiles.finders.AppDirectoriesFinder',
# other finders..
'compressor.finders.CompressorFinder',
)
CACHES = {
'default': {
'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
'LOCATION': '127.0.0.1:11211',
}
}
# Internationalization
# https://docs.djangoproject.com/en/2.2/topics/i18n/
LANGUAGE_CODE = 'en-us'
TIME_ZONE = 'UTC'
USE_I18N = True
USE_L10N = True
USE_TZ = True
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/2.2/howto/static-files/
STATIC_URL = '/static/'
STATICFILES_DIRS = [os.path.join(BASE_DIR, 'static'),]
STATIC_ROOT = os.path.join(BASE_DIR,"static_root")
MEDIA_URL = '/media/'
MEDIA_ROOT = os.path.join(BASE_DIR, 'media')
LOGIN_REDIRECT_URL = "home:index"
LOGOUT_REDIRECT_URL = "home:index"
environment:
- DB_HOST=db
- DB_NAME=app
- DB_USER=postgres
- DB_PASS=supersecretpassword
I see that you are passing the environment variables via docker-compose like this. This is fine when the container is running the command directly inside the shell.
However, when putting it inside the crontab, the cronjob will run your command in a separate fresh shell with no environment being passed in at all.
To work around this problem, you can create a separate shell script:
cat <<EOF > /temp/script.sh
#!/bin/bash
export DB_HOST=db
export DB_NAME=app
export DB_USER=postgres
export DB_PASS=supersecretpassword
cd /tourMamaRoot/tourMama/cronjob && fetch_news.py >> /var/log/cron.log 2>&1
EOF
chmod +x /temp/script.sh
and edit your crontab like this:
* * * * * /temp/script.sh