Update MELPA to build both unstable and stable. (#5612)

- Combine stable and unstable builds into the same rotation so one
builds after the other.

- Move all the services to docker containers,
- nginx: contains everything for handling http traffic including log
  rotation and ssl certs.
- syncer: manages syncing all the html files to the latest build
  information.
- logprocessor: processes the http logs.
- rsyncd: rsync for melpa repo.
pull/5645/head
Donald Curtis 2018-07-27 10:44:38 -05:00 committed by GitHub
parent b623ce3e2b
commit 176e0e6410
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
48 changed files with 483 additions and 427 deletions

8
.gitignore vendored
View File

@ -4,6 +4,8 @@
/.cask
/.ecukes-failing-scenarios
/config.mk
/download_log_stable.db
/download_log_stable.json.gz
/download_log.db
/download_log.json.gz
/elpa
@ -14,6 +16,12 @@
/html/download_counts.json
/html/recipes.json
/html/updates.rss
/html-stable/.last-build-duration
/html-stable/archive.json
/html-stable/build-status.json
/html-stable/download_counts.json
/html-stable/recipes.json
/html-stable/updates.rss
/packages-stable/*
/packages/*
/process_log.pid

View File

@ -9,10 +9,13 @@ PKGDIR := packages
RCPDIR := recipes
HTMLDIR := html
WORKDIR := working
WEBROOT := $$HOME/www
SLEEP ?= 0
SANDBOX := sandbox
STABLE ?= nil
ifneq ($(STABLE), nil)
PKGDIR := packages-stable
HTMLDIR := html-stable
endif
LISP_CONFIG ?= '(progn\
(setq package-build-working-dir "$(TOP)/$(WORKDIR)/")\
@ -51,7 +54,7 @@ clean-packages:
clean-json:
@echo " • Removing json files ..."
@-rm -vf html/archive.json html/recipes.json
@-rm -vf $(HTMLDIR)/archive.json $(HTMLDIR)/recipes.json
clean-sandbox:
@echo " • Removing sandbox files ..."
@ -60,13 +63,6 @@ clean-sandbox:
rmdir '$(SANDBOX)'; \
fi
sync:
@echo " • Synchronizing files ..."
@rsync -avz --delete $(PKGDIR)/ $(WEBROOT)/packages
@rsync -avz --safe-links --delete $(HTMLDIR)/* $(WEBROOT)/
@chmod -R go+rx $(WEBROOT)/packages/*
pull-package-build:
git subtree pull --squash -P package-build package-build master
@ -93,7 +89,15 @@ html/recipes.json: $(RCPDIR)/.dirstamp
@echo " • Building $@ ..."
@$(EVAL) '(package-build-recipe-alist-as-json "html/recipes.json")'
json: html/archive.json html/recipes.json
html-stable/archive.json: $(PKGDIR)/archive-contents
@echo " • Building $@ ..."
@$(EVAL) '(package-build-archive-alist-as-json "html-stable/archive.json")'
html-stable/recipes.json: $(RCPDIR)/.dirstamp
@echo " • Building $@ ..."
@$(EVAL) '(package-build-recipe-alist-as-json "html-stable/recipes.json")'
json: $(HTMLDIR)/archive.json $(HTMLDIR)/recipes.json
$(RCPDIR)/.dirstamp: .FORCE
@[[ ! -e $@ || "$$(find $(@D) -newer $@ -print -quit)" != "" ]] \

View File

@ -1,12 +0,0 @@
# MELPA Config Files
This directory contains config files for applications needed to
maintain the MELPA infrastructure.
Currently this directory contains local config files for *nginx* and
*logrotate* which are run by the user processes.
The `etc` directory contains system-level configs required to
configure the main `runit` process to run user services as well as the
main *nginx* proxy.

View File

@ -1,3 +0,0 @@
#!/bin/sh -e
exec 2>&1
exec chpst -u melpa runsvdir /home/melpa/service 'log: ...........................................................................................................................................................................................................................................................................................................................................................................................................'

View File

@ -1,19 +0,0 @@
/home/melpa/log/melpa*.log {
daily
create 0640 melpa melpa
compress
dateext
missingok
missingok
notifempty
rotate 36500
sharedscripts
prerotate
if [ -d /etc/logrotate.d/httpd-prerotate ]; then \
run-parts /etc/logrotate.d/httpd-prerotate; \
fi; \
endscript
postrotate
[ ! -f /home/melpa/var/run/nginx.pid ] || kill -USR1 `cat /home/melpa/var/run/nginx.pid`
endscript
}

View File

@ -0,0 +1,5 @@
FROM debian:unstable
RUN apt-get update
RUN DEBIAN_FRONTEND=noninteractive apt-get --yes install emacs25-nox git make mercurial ruby
WORKDIR /mnt/store/melpa
CMD docker/builder/run.sh

View File

@ -14,7 +14,7 @@ function build_all {
PATH=$HOME/.cabal/bin:$HOME/usr/bin:$HOME/bin:$PATH
## run the script
cd "$(dirname "${0}")/.."
cd "/mnt/store/melpa"
make -j4 $(grep --files-without-match ':fetcher\s*wiki' $(find recipes/ -type f -not -name python-info -and -not -name python3-info | sort)) &
wait

72
docker/builder/run.sh Executable file
View File

@ -0,0 +1,72 @@
#!/bin/bash -e
MELPA_REPO=/mnt/store/melpa
cd "${MELPA_REPO}"
MELPA_BRANCH=$( git rev-parse --abbrev-ref HEAD )
STATUS_JSON=${MELPA_REPO}/html/build-status.json
LAST_DURATION_FILE=${MELPA_REPO}/html/.last-build-duration
STABLE_STATUS_JSON=${MELPA_REPO}/html-stable/build-status.json
STABLE_LAST_DURATION_FILE=${MELPA_REPO}/html-stable/.last-build-duration
## update MELPA repo
git fetch origin
git reset --hard "origin/${MELPA_BRANCH}"
git pull origin "${MELPA_BRANCH}"
echo
update_json() {
cat <<EOF > $BUILD_STATUS_JSON
{
"started": $BUILD_STARTED,
"completed": ${BUILD_COMPLETED-null},
"next": ${BUILD_NEXT-null},
"duration": ${BUILD_DURATION-null}
}
EOF
echo "Writing $BUILD_STATUS_JSON"
cat "$BUILD_STATUS_JSON"
}
flux_capacitor() {
if [ -f "$BUILD_LAST_DURATION_FILE" ]; then
BUILD_DURATION=$(cat "$BUILD_LAST_DURATION_FILE")
fi
BUILD_STARTED=$(date "+%s")
update_json
# Build all the packages.
docker/builder/parallel_build_all
# Store completed date
BUILD_COMPLETED=$(date "+%s")
BUILD_DURATION=$((BUILD_COMPLETED - BUILD_STARTED))
echo -n "$BUILD_DURATION" > $BUILD_LAST_DURATION_FILE
BUILD_NEXT=$((BUILD_COMPLETED + BUILD_DELAY))
update_json
}
unset STABLE
BUILD_STATUS_JSON=${STATUS_JSON}
BUILD_LAST_DURATION_FILE=${LAST_DURATION_FILE}
if [ -f "$STABLE_LAST_DURATION_FILE" ]; then
BUILD_DELAY=$(cat "$STABLE_LAST_DURATION_FILE")
fi
flux_capacitor
# stable build
export STABLE=t
BUILD_STATUS_JSON=${STABLE_STATUS_JSON}
BUILD_LAST_DURATION_FILE=${STABLE_LAST_DURATION_FILE}
if [ -f "$LAST_DURATION_FILE" ]; then
BUILD_DELAY=$(cat "$LAST_DURATION_FILE")
fi
flux_capacitor

44
docker/docker-compose.yml Normal file
View File

@ -0,0 +1,44 @@
version: "3"
services:
nginx:
build: ./nginx
image: melpa/nginx:v1
restart: unless-stopped
volumes:
- /mnt/store/melpa:/mnt/store/melpa:ro
- /mnt/store/letsencrypt:/etc/letsencrypt
- /mnt/store/log:/mnt/store/log
- /mnt/store/log-stable:/mnt/store/log-stable
ports:
- "80:80"
- "443:443"
builder:
build: ./builder
image: melpa/builder:v1
restart: unless-stopped
volumes:
- /mnt/store/melpa:/mnt/store/melpa
logprocessor:
build: ./logprocessor
image: melpa/logprocessor:v1
restart: unless-stopped
volumes:
- /mnt/db:/mnt/db
- /mnt/store/melpa:/mnt/store/melpa
- /mnt/store/log:/mnt/store/log:ro
- /mnt/store/log-stable:/mnt/store/log-stable:ro
syncer:
build: ./syncer
image: melpa/syncer:v1
restart: unless-stopped
volumes:
- /mnt/store/melpa:/mnt/store/melpa
rsyncd:
build: ./rsyncd
image: melpa/rsyncd:v1
restart: unless-stopped
ports:
- "873:873"
volumes:
- /mnt/store/melpa/packages:/packages:ro
- /mnt/store/melpa/packages-stable:/packages-stable:ro

View File

@ -0,0 +1,5 @@
FROM debian:unstable
RUN apt-get update
RUN DEBIAN_FRONTEND=noninteractive apt-get --yes install python
WORKDIR /mnt/store/melpa
CMD docker/logprocessor/run.sh

View File

@ -13,7 +13,10 @@ import tempfile
import sqlite3
from operator import or_
LOGFILE = "/home/melpa/log/melpa.access.log"
STABLE = os.getenv("STABLE")
LOGFILE = "/mnt/store/log/melpa.access.log"
if STABLE:
LOGFILE = "/mnt/store/log-stable/melpa.access.log"
LOGREGEX = r'(?P<ip>[\d.]+) [ -]+ \[(?P<date>[\w/: +-]+)\] ' \
r'"GET /packages/(?P<package>[^ ]+)-[0-9.]+.(?:el|tar) ' \
r'HTTP/\d.\d" 200'
@ -25,8 +28,8 @@ def json_handler(obj):
elif isinstance(obj, set):
return list(obj)
raise TypeError(
'Object of type {0} with value {0} is not JSON serializable'.format(
type(obj), repr(obj)))
'Object of type {0} with value {1} is not JSON serializable'.format(
type(obj), repr(obj)))
def json_dump(data, jsonfile, indent=None):
@ -58,6 +61,7 @@ def ip_to_number(ip):
return reduce(or_, ((int(n) << (i*8)) for i, n in enumerate(
reversed(ip.split('.')))), 0)
def parse_logfile(logfilename, curs):
"""
"""
@ -77,9 +81,6 @@ def parse_logfile(logfilename, curs):
# Convert ips to four character strings.
ip = match.group('ip')
dtstamp = int(time.mktime(
datetime.strptime(match.group('date').split()[0],
"%d/%b/%Y:%H:%M:%S").timetuple()))
pkg = match.group('package')
curs.execute("INSERT OR IGNORE INTO pkg_ip VALUES (?, ?)", (pkg, ip))
@ -96,28 +97,17 @@ def main():
help="Log files to parse.", default=[LOGFILE])
args = parser.parse_args()
pid = str(os.getpid())
pidfile = os.path.join(os.path.join(tempfile.gettempdir(), "process_log.pid"))
db_filename = "/mnt/db/download_log.db"
if STABLE:
db_filename = "/mnt/db/download_log_stable.db"
if os.access(pidfile, os.F_OK):
running_pid = open(pidfile, "r").readline()
try:
os.kill(int(running_pid), 0)
print "Process {0} currently running.".format(running_pid)
return 1
except OSError:
print "Stale lockfile."
os.unlink(pidfile)
file(pidfile, 'w').write(pid)
new_db = not os.path.exists("download_log.db")
conn = sqlite3.connect("download_log.db")
new_db = not os.path.exists(db_filename)
conn = sqlite3.connect(db_filename)
curs = conn.cursor()
if new_db:
sys.stdout.write("creating database...\n")
curs.execute('''CREATE TABLE pkg_ip (package, ip, PRIMARY KEY (package, ip))''')
curs.execute(
'''CREATE TABLE pkg_ip (package, ip, PRIMARY KEY (package, ip))''')
conn.commit()
# parse each parameter
@ -130,10 +120,14 @@ def main():
conn.commit()
# calculate current package totals
pkgcount = {p: c for p,c in curs.execute("SELECT package, count(ip) FROM pkg_ip GROUP BY 1")}
json_dump(pkgcount, open("html/download_counts.json", 'w'), indent=1)
pkgcount = {p: c for p, c in curs.execute(
"SELECT package, count(ip) FROM pkg_ip GROUP BY 1")}
if STABLE:
json_dump(pkgcount, open(
"html-stable/download_counts.json", 'w'), indent=1)
else:
json_dump(pkgcount, open("html/download_counts.json", 'w'), indent=1)
os.unlink(pidfile)
return 0

11
docker/logprocessor/run.sh Executable file
View File

@ -0,0 +1,11 @@
#!/bin/bash -e
MELPA_REPO=/mnt/store/melpa
cd "${MELPA_REPO}"
unset STABLE
/usr/bin/python ${MELPA_REPO}/docker/logprocessor/process_log.py
export STABLE=t
/usr/bin/python ${MELPA_REPO}/docker/logprocessor/process_log.py

6
docker/nginx/Dockerfile Normal file
View File

@ -0,0 +1,6 @@
FROM nginx:alpine
RUN apk add --no-cache certbot openssl logrotate
COPY default.conf /etc/nginx/conf.d/
COPY logrotate /etc/logrotate
WORKDIR /mnt/store/melpa
CMD docker/nginx/run.sh

152
docker/nginx/default.conf Normal file
View File

@ -0,0 +1,152 @@
# sendfile on;
tcp_nopush on;
tcp_nodelay on;
# keepalive_timeout 65;
types_hash_max_size 2048;
server_tokens off;
# server_names_hash_bucket_size 64;
charset utf-8;
#include /etc/nginx/mime.types;
#default_type application/octet-stream;
chunked_transfer_encoding off;
gzip on;
gzip_disable "msie6|Emacs";
gzip_vary on;
gzip_proxied any;
gzip_comp_level 6;
gzip_buffers 16 8k;
gzip_http_version 1.1;
gzip_types text/plain text/xhtml text/css text/js text/csv application/javascript application/x-javascript application/json application/xml text/xml application/atom+xml application/rss+xml;
client_max_body_size 10m;
client_body_buffer_size 128k;
server {
server_name melpa.milkbox.net;
rewrite ^ $scheme://melpa.org$request_uri? permanent;
}
server {
server_name melpa-stable.milkbox.net;
server_name hiddencameras.milkbox.net;
rewrite ^ $scheme://stable.melpa.org$request_uri? permanent;
}
server {
listen 80;
listen 443 ssl;
server_name melpa.org www.melpa.org test.melpa.org;
root /mnt/store/melpa/html;
access_log /mnt/store/log/melpa.access.log combined;
error_log /mnt/store/log/melpa.error.log info;
ssl_certificate /etc/letsencrypt/live/melpa.org/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/melpa.org/privkey.pem;
# Lock down ciphers / SSL versions to attain a good security rating
# https://www.ssllabs.com/ssltest/analyze.html?d=melpa.org
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_prefer_server_ciphers on;
ssl_dhparam /etc/letsencrypt/ssl/dhparam.pem;
ssl_ciphers 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA';
ssl_session_timeout 1d;
ssl_session_cache shared:SSL:50m;
ssl_stapling on;
ssl_stapling_verify on;
add_header Strict-Transport-Security max-age=15768000;
location ~ /.well-known {
allow all;
root /usr/share/nginx/html;
}
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root /var/www/nginx-default;
}
location = /packages/archive-contents {
default_type text/plain;
}
location ~ ^/packages/.*\.el {
default_type text/plain;
}
location ~ ^/packages/.*\.svg {
add_header Cache-Control no-cache;
}
location ^~ /.well-known/acme-challenge/ {
default_type "text/plain";
root /tmp/letsencrypt-auto;
}
### Hide /acme-challenge subdirectory and return 404 on all requests.
location = /.well-known/acme-challenge/ {
return 404;
}
}
server {
listen 80;
listen 443 ssl;
server_name stable.melpa.org stable-test.melpa.org;
root /mnt/store/melpa/html-stable;
access_log /mnt/store/log-stable/melpa.access.log combined;
error_log /mnt/store/log-stable/melpa.error.log info;
ssl_certificate /etc/letsencrypt/live/melpa.org/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/melpa.org/privkey.pem;
# Lock down ciphers / SSL versions to attain a good security rating
# https://www.ssllabs.com/ssltest/analyze.html?d=melpa.org
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_prefer_server_ciphers on;
ssl_dhparam /etc/letsencrypt/ssl/dhparam.pem;
ssl_ciphers 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA';
ssl_session_timeout 1d;
ssl_session_cache shared:SSL:50m;
ssl_stapling on;
ssl_stapling_verify on;
add_header Strict-Transport-Security max-age=15768000;
location ~ /.well-known {
allow all;
root /usr/share/nginx/html;
}
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root /var/www/nginx-default;
}
location = /packages/archive-contents {
default_type text/plain;
}
location ~ ^/packages/.*\.el {
default_type text/plain;
}
location ~ ^/packages/.*\.svg {
add_header Cache-Control no-cache;
}
location ^~ /.well-known/acme-challenge/ {
default_type "text/plain";
root /tmp/letsencrypt-auto;
}
### Hide /acme-challenge subdirectory and return 404 on all requests.
location = /.well-known/acme-challenge/ {
return 404;
}
}

39
docker/nginx/logrotate Normal file
View File

@ -0,0 +1,39 @@
/mnt/store/log/melpa*.log {
daily
create 0640 root root
compress
dateext
missingok
missingok
notifempty
rotate 36500
sharedscripts
prerotate
if [ -d /etc/logrotate.d/httpd-prerotate ]; then \
run-parts /etc/logrotate.d/httpd-prerotate; \
fi; \
endscript
postrotate
[ ! -f /var/run/nginx.pid ] || kill -USR1 `cat /var/run/nginx.pid`
endscript
}
/mnt/store/log-stable/melpa*.log {
daily
create 0640 root root
compress
dateext
missingok
missingok
notifempty
rotate 36500
sharedscripts
prerotate
if [ -d /etc/logrotate.d/httpd-prerotate ]; then \
run-parts /etc/logrotate.d/httpd-prerotate; \
fi; \
endscript
postrotate
[ ! -f /var/run/nginx.pid ] || kill -USR1 `cat /var/run/nginx.pid`
endscript
}

50
docker/nginx/run.sh Executable file
View File

@ -0,0 +1,50 @@
#!/bin/sh
mkdir -p /tmp/letsencrypt-auto
mkdir -p /etc/letsencrypt/ssl 2> /dev/null
[ -f /etc/letsencrypt/ssl/dhparam.pem ] || openssl dhparam -out /etc/letsencrypt/ssl/dhparam.pem 2048
# bootstrap certbot
[ -d /etc/letsencrypt/live/test.melpa.org ] || certbot certonly \
--standalone \
--agree-tos \
--email dcurtis@gmail.com \
--non-interactive \
--rsa-key-size 4096 \
--force-renewal \
--expand \
-d melpa.org \
-d stable.melpa.org \
-d test.melpa.org \
-d stable-test.melpa.org
nginx -g 'daemon off;' &
nginx_pid=$!
while true; do
echo "goodnight world..."
# sleep for 1 day
sleep 86400
echo "rotate them logs"
logrotate /etc/logrotate
echo "maybe refreshing certbot..."
mkdir -p /tmp/letsencrypt-auto
certbot certonly \
--webroot \
--webroot-path /tmp/letsencrypt-auto \
--agree-tos \
--email dcurtis@gmail.com \
--non-interactive \
--rsa-key-size 4096 \
--expand \
-d melpa.org \
-d stable.melpa.org \
-d test.melpa.org \
-d stable-test.melpa.org
echo "restarting nginx..."
kill -HUP $nginx_pid
done

4
docker/rsyncd/Dockerfile Normal file
View File

@ -0,0 +1,4 @@
FROM nginx:alpine
RUN apk add --no-cache rsync
COPY rsyncd.conf /etc/
CMD /usr/bin/rsync --no-detach --daemon --config /etc/rsyncd.conf

15
docker/rsyncd/rsyncd.conf Normal file
View File

@ -0,0 +1,15 @@
max connections = 1
log file = /var/log/rsync.log
timeout = 300
[packages]
comment = MELPA packages
path = /packages
read only = yes
list = yes
[packages-stable]
comment = MELPA stable packages
path = /packages-stable
read only = yes
list = yes

5
docker/syncer/Dockerfile Normal file
View File

@ -0,0 +1,5 @@
FROM debian:unstable
RUN apt-get update
RUN DEBIAN_FRONTEND=noninteractive apt-get --yes install emacs25-nox git make mercurial ruby
WORKDIR /mnt/store/melpa
CMD docker/syncer/run.sh

17
docker/syncer/run.sh Executable file
View File

@ -0,0 +1,17 @@
#!/bin/bash -e
MELPA_REPO=/mnt/store/melpa
cd "${MELPA_REPO}"
unset STABLE
make cleanup
make html
export STABLE=t
make cleanup
make html
# Sync every 5 minutes.
sleep 5m

1
html-stable/Makefile Symbolic link
View File

@ -0,0 +1 @@
../html/Makefile

1
html-stable/css Symbolic link
View File

@ -0,0 +1 @@
../html/css

1
html-stable/donate.png Symbolic link
View File

@ -0,0 +1 @@
../html/donate.png

1
html-stable/favicon.ico Symbolic link
View File

@ -0,0 +1 @@
../html/favicon.ico

1
html-stable/index.html Symbolic link
View File

@ -0,0 +1 @@
../html/index.html

1
html-stable/js Symbolic link
View File

@ -0,0 +1 @@
../html/js

1
html-stable/jslicense.html Symbolic link
View File

@ -0,0 +1 @@
../html/jslicense.html

1
html-stable/packages Symbolic link
View File

@ -0,0 +1 @@
../packages-stable

1
html-stable/partials Symbolic link
View File

@ -0,0 +1 @@
../html/partials

1
html-stable/robots.txt Symbolic link
View File

@ -0,0 +1 @@
../html/robots.txt

1
html-stable/updates.rss.erb Symbolic link
View File

@ -0,0 +1 @@
../html/updates.rss.erb

View File

@ -524,7 +524,8 @@
// Changing the appearance of the MELPA Stable page
//////////////////////////////////////////////////////////////////////////////
melpa.stable = m.prop(window.location.host === 'stable.melpa.org');
melpa.stable = m.prop(window.location.host === 'stable.melpa.org' ||
window.location.host === 'stable-test.melpa.org');
melpa.archivename = {};
melpa.archivename.controller = function() {
this.archiveName = function() {

1
packages-stable/index.html Symbolic link
View File

@ -0,0 +1 @@
../packages/index.html

View File

@ -1,39 +0,0 @@
#!/bin/bash
# \curl -L https://raw.github.com/melpa/melpa/master/scripts/bootstrap | bash
SUDOENV='DEBIAN_FRONTEND=noninteractive'
cd ${HOME}
#sudo ${SUDOENV} add-apt-repository -y ppa:cassou/emacs
sudo ${SUDOENV} apt-get -y update
sudo ${SUDOENV} apt-get -y upgrade
sudo ${SUDOENV} apt-get -y install \
curl \
emacs24 emacs24-el emacs24-common-non-dfsg \
git \
make \
mercurial \
nginx \
runit \
texinfo \
tmux
# build the log stuff
mkdir -p ~/log
mkdir -p ~/var/run
mkdir -p ~/var/lib/logrotate
mkdir -p ~/etc
# ruby is used for templating
\curl -L https://get.rvm.io | bash -s stable
PATH=$PATH:$HOME/.rvm/bin
source "$HOME/.rvm/scripts/rvm"
rvm requirements
rvm install ruby-1.9.3
rvm use ruby-1.9.3 --default
echo "source ~/.profile" >> .bash_profile
git clone http://github.com/melpa/melpa
sudo reboot

View File

@ -1,3 +0,0 @@
#!/bin/sh -e
[ -d main ] || mkdir -p main
exec svlogd -tt main

View File

@ -1,6 +0,0 @@
#!/bin/bash -e
SCRIPTPATH=$( cd "$(dirname "$0")" ; pwd -P )
export STABLE=t
. "$SCRIPTPATH/../builder/run"

View File

@ -1,3 +0,0 @@
#!/bin/sh -e
[ -d main ] || mkdir -p main
exec svlogd -tt main

View File

@ -1,54 +0,0 @@
#!/bin/bash -e
SCRIPTPATH=$( cd "$(dirname "$0")" ; pwd -P )
export HOME=$( cd "${SCRIPTPATH}/../../.." ; pwd -P )
MELPA_REPO="${SCRIPTPATH}/../.."
cd "${MELPA_REPO}"
MELPA_BRANCH=$( git rev-parse --abbrev-ref HEAD )
BUILD_STATUS_JSON=${MELPA_REPO}/html/build-status.json
BUILD_LAST_DURATION_FILE=${MELPA_REPO}/html/.last-build-duration
# Amount of time to sleep between rebuilds
BUILD_DELAY=$((10 * 60))
## update MELPA repo
git fetch origin
git reset --hard "origin/${MELPA_BRANCH}"
git pull origin "${MELPA_BRANCH}"
echo
if [ -f "$BUILD_LAST_DURATION_FILE" ]; then
BUILD_DURATION=$(cat "$BUILD_LAST_DURATION_FILE")
fi
update_json() {
cat <<EOF > $BUILD_STATUS_JSON
{
"started": $BUILD_STARTED,
"completed": ${BUILD_COMPLETED-null},
"next": ${BUILD_NEXT-null},
"duration": ${BUILD_DURATION-null}
}
EOF
echo "Writing $BUILD_STATUS_JSON"
cat "$BUILD_STATUS_JSON"
}
BUILD_STARTED=$(date "+%s")
update_json
# Build all the packages.
scripts/parallel_build_all
# Store completed date
BUILD_COMPLETED=$(date "+%s")
BUILD_DURATION=$((BUILD_COMPLETED - BUILD_STARTED))
echo -n "$BUILD_DURATION" > $BUILD_LAST_DURATION_FILE
BUILD_NEXT=$((BUILD_COMPLETED + BUILD_DELAY))
update_json
# Sleep for a while before rebuilding.
sleep $BUILD_DELAY

View File

@ -1,3 +0,0 @@
#!/bin/bash -e
exec /usr/local/bin/caddy --agree --log /home/melpa/log/caddy.log --conf /home/melpa/Caddyfile

View File

@ -1,91 +0,0 @@
worker_processes 4;
pid /home/melpa/var/run/nginx.pid;
daemon off;
events {
worker_connections 768;
}
http {
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
server_tokens off;
# server_names_hash_bucket_size 64;
charset utf-8;
include /etc/nginx/mime.types;
default_type application/octet-stream;
access_log /home/melpa/log/melpa.access.log combined;
error_log /home/melpa/log/melpa.error.log info;
chunked_transfer_encoding off;
gzip on;
gzip_disable "msie6|Emacs";
gzip_vary on;
gzip_proxied any;
gzip_comp_level 6;
gzip_buffers 16 8k;
gzip_http_version 1.1;
gzip_types text/plain text/xhtml text/css text/js text/csv application/javascript application/x-javascript application/json application/xml text/xml application/atom+xml application/rss+xml;
client_max_body_size 10m;
client_body_buffer_size 128k;
server {
server_name melpa-stable.milkbox.net;
server_name hiddencameras.milkbox.net;
rewrite ^ $scheme://stable.melpa.org$request_uri? permanent;
}
server {
listen 80;
listen 443 ssl;
server_name localhost stable.melpa.org;
root /home/melpa/melpa/html;
ssl_certificate /etc/letsencrypt/live/stable.melpa.org/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/stable.melpa.org/privkey.pem;
# Lock down ciphers / SSL versions to attain a good security rating
# https://www.ssllabs.com/ssltest/analyze.html?d=melpa.org
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_prefer_server_ciphers on;
ssl_dhparam /etc/ssl/certs/dhparam.pem;
ssl_ciphers 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA';
ssl_session_timeout 1d;
ssl_session_cache shared:SSL:50m;
ssl_stapling on;
ssl_stapling_verify on;
add_header Strict-Transport-Security max-age=15768000;
location ~ /.well-known {
allow all;
root /usr/share/nginx/html;
}
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root /var/www/nginx-default;
}
location = /packages/archive-contents {
default_type text/plain;
}
location ~ ^/packages/.*\.el {
default_type text/plain;
}
location ~ ^/packages/.*\.svg {
add_header Cache-Control no-cache;
}
}
}

View File

@ -1,3 +0,0 @@
#!/bin/bash -e
/usr/bin/authbind /usr/sbin/nginx -g 'error_log /dev/null crit;' -c /home/melpa/service/nginx-stable/config

View File

@ -1,91 +0,0 @@
worker_processes 4;
pid /home/melpa/var/run/nginx.pid;
daemon off;
events {
worker_connections 768;
}
http {
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
server_tokens off;
# server_names_hash_bucket_size 64;
charset utf-8;
include /etc/nginx/mime.types;
default_type application/octet-stream;
access_log /home/melpa/log/melpa.access.log combined;
error_log /home/melpa/log/melpa.error.log info;
chunked_transfer_encoding off;
gzip on;
gzip_disable "msie6|Emacs";
gzip_vary on;
gzip_proxied any;
gzip_comp_level 6;
gzip_buffers 16 8k;
gzip_http_version 1.1;
gzip_types text/plain text/xhtml text/css text/js text/csv application/javascript application/x-javascript application/json application/xml text/xml application/atom+xml application/rss+xml;
client_max_body_size 10m;
client_body_buffer_size 128k;
server {
server_name melpa.milkbox.net;
rewrite ^ $scheme://melpa.org$request_uri? permanent;
}
server {
listen 80;
listen 443 ssl;
server_name localhost melpa.org www.melpa.org;
root /home/melpa/melpa/html;
ssl_certificate /etc/letsencrypt/live/melpa.org/fullchain.pem;
ssl_certificate_key /etc/letsencrypt/live/melpa.org/privkey.pem;
# Lock down ciphers / SSL versions to attain a good security rating
# https://www.ssllabs.com/ssltest/analyze.html?d=melpa.org
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_prefer_server_ciphers on;
ssl_dhparam /etc/ssl/certs/dhparam.pem;
ssl_ciphers 'ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:CAMELLIA:DES-CBC3-SHA:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA';
ssl_session_timeout 1d;
ssl_session_cache shared:SSL:50m;
ssl_stapling on;
ssl_stapling_verify on;
add_header Strict-Transport-Security max-age=15768000;
location ~ /.well-known {
allow all;
root /usr/share/nginx/html;
}
error_page 500 502 503 504 /50x.html;
location = /50x.html {
root /var/www/nginx-default;
}
location = /packages/archive-contents {
default_type text/plain;
}
location ~ ^/packages/.*\.el {
default_type text/plain;
}
location ~ ^/packages/.*\.svg {
add_header Cache-Control no-cache;
}
}
}

View File

@ -1,3 +0,0 @@
#!/bin/bash -e
/usr/bin/authbind /usr/sbin/nginx -g 'error_log /dev/null crit;' -c /home/melpa/service/nginx/config

View File

@ -1,11 +0,0 @@
#!/bin/bash -e
SCRIPTPATH=$( cd "$(dirname "$0")" ; pwd -P )
MELPA_REPO="${SCRIPTPATH}/../.."
cd "${MELPA_REPO}"
/usr/bin/python ${MELPA_REPO}/scripts/process_log.py
/usr/sbin/logrotate -s ${MELPA_REPO}/../var/lib/logrotate/status ${MELPA_REPO}/cfg/logrotate
sleep 6h

View File

@ -1,24 +0,0 @@
#!/bin/bash -e
# Service for reporting missing and old packages.
SCRIPTPATH=$( cd "$(dirname "$0")" ; pwd -P )
MELPA_REPO="${SCRIPTPATH}/../.."
cd "${MELPA_REPO}"
BODY=`mktemp`
echo "To: MP <dcurtis@milkbox.net>" > ${BODY}
echo "Subject: [MELPA] `date "+%Y%m%d %H:%M %z"`" >> ${BODY}
echo >> ${BODY}
# echo "# Old Packages" >> ${BODY}
# echo >> ${BODY}
# scripts/expired >> ${BODY}
echo >> ${BODY}
echo "# Missing Packages" >> ${BODY}
echo >> ${BODY}
scripts/missing >> ${BODY}
/usr/sbin/sendmail -t < ${BODY}
sleep 12h

View File

@ -1,6 +0,0 @@
#!/bin/bash -e
SCRIPTPATH=$( cd "$(dirname "$0")" ; pwd -P )
export STABLE=t
. "$SCRIPTPATH/../syncer/run"

View File

@ -1,3 +0,0 @@
#!/bin/sh -e
[ -d main ] || mkdir -p main
exec svlogd -tt main

View File

@ -1,14 +0,0 @@
#!/bin/bash -e
SCRIPTPATH=$( cd "$(dirname "$0")" ; pwd -P )
MELPA_REPO="${SCRIPTPATH}/../.."
cd "${MELPA_REPO}"
[[ -s "../.rvm/scripts/rvm" ]] && source "../.rvm/scripts/rvm"
make cleanup
make json
make html
# Sync every 5 minutes.
sleep 5m