Browse Source

Add backup-volumes tool

master
Sean Johnson 2 years ago
parent
commit
c79745c41f
  1. 24
      pipeline.yml
  2. 3
      resource/Dockerfile
  3. 185
      resource/tools/backup-volumes.py
  4. 13
      settings.yml

24
pipeline.yml

@ -23,12 +23,12 @@ resources:
tag: (( grab meta.upstream.golang.tag ))
.: (( inject meta.upstream.golang.auth ))
- name: convoy-vault
- name: convoy
type: registry-image
icon: wall
source:
repository: (( grab meta.image.convoy-vault.repo ))
tag: (( grab meta.image.convoy-vault.tag ))
repository: (( grab meta.image.convoy.repo ))
tag: (( grab meta.image.convoy.tag ))
.: (( inject meta.registry.auth ))
- name: source
@ -37,16 +37,8 @@ resources:
source:
.: (( inject meta.source ))
- name: vault-init
type: registry-image
icon: docker
source:
repository: (( grab meta.upstream.vault-init.repo ))
tag: (( grab meta.upstream.vault-init.tag ))
.: (( inject meta.upstream.vault-init.auth ))
jobs:
- name: "convoy-vault"
- name: "convoy"
public: true
plan:
- get: commons
@ -55,9 +47,7 @@ jobs:
- get: alpine
trigger: true
- get: golang
- get: vault-init
trigger: true
- task: "build convoy-vault image"
- task: "build convoyimage"
file: (( grab meta.tasks.img-build-oci ))
privileged: true
input_mapping: {context: source}
@ -67,14 +57,12 @@ jobs:
BUILD_ARG_CONVOY_VERSION: (( grab meta.upstream.convoy.version ))
BUILD_ARG_GOLANG_REPO: (( grab meta.upstream.golang.repo ))
BUILD_ARG_GOLANG_VERSION: (( grab meta.upstream.golang.tag ))
BUILD_ARG_VAULT_INIT_REPO: (( grab meta.upstream.vault-init.repo ))
BUILD_ARG_VAULT_INIT_VERSION: (( grab meta.upstream.vault-init.tag ))
CONTEXT: (( grab meta.image.convoy-vault.context ))
- task: "write image tags"
file: (( grab meta.tasks.img-write-tags ))
params:
TAGS: (( grab meta.upstream.convoy.version ))
- put: convoy-vault
- put: convoy
params:
image: image/image.tar
additional_tags: metadata/additional_tags

3
resource/Dockerfile

@ -30,7 +30,8 @@ FROM ${ALPINE_REPO}:${ALPINE_VERSION}
COPY --from=vault-init /bin/vault-init /bin/vault-init
COPY --from=build /go/bin/convoy /bin/convoy
ADD tools /tools
RUN apk add --no-cache bash jq lvm2-libs
RUN apk add --no-cache bash jq lvm2-libs python3
ENTRYPOINT ["/bin/vault-init", "-O", "/bin/convoy"]

185
resource/tools/backup-volumes.py

@ -0,0 +1,185 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import asyncio
import json
import os
import re
import shutil
import sys
from datetime import datetime
from typing import Iterator, List, Optional, Sequence, Union
_CONVOY_DATE_FMT = "%a %b %d %H:%M:%S %z %Y"
BACKUP_DESTINATION = os.getenv("BACKUP_DESTINATION", None)
BACKUP_INTERVAL = int(os.getenv("BACKUP_INTERVAL", 86400))
CONCURRENT_SNAPSHOTS = int(os.getenv("CONCURRENT_SNAPSHOTS", 4))
CONVOY_BIN = shutil.which("convoy") or os.getenv("CONVOY_BIN")
CONVOY_SOCKET = os.getenv("CONVOY_SOCKET", "/var/run/convoy/convoy.sock")
KEEP_LAST_BACKUPS = int(os.getenv("KEEP_LAST_BACKUPS", 7))
KEEP_LAST_SNAPSHOTS = int(os.getenv("KEEP_LAST_SNAPSHOTS", 7))
ONLY_MATCHING = re.compile(os.getenv("ONLY_MATCHING", r"^.*$"))
def timestamp_sec() -> str:
return datetime.now().strftime("%Y-%m-%dT%H-%M-%SZ%z")
def timestamp_micros() -> str:
return datetime.now().strftime("%Y-%m-%dT%H-%M-%S.%fZ%z")
def chunked_iter(items: Sequence, size: int) -> Iterator[List[List]]:
""" Breaks a sequence into chunks of `size`.
"""
if size == 0:
raise ValueError("Chunk size must not be zero")
for index in range(0, len(items), size):
yield items[index : index + size]
def log_stdout(message, **context):
message = {
"timestamp": timestamp_micros(),
"message": message,
"context": context,
}
sys.stdout.write(json.dumps(message) + "\n")
async def convoy(*args) -> asyncio.subprocess.Process:
""" Call the `convoy` binary.
"""
return await asyncio.create_subprocess_exec(
CONVOY_BIN, *args,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.DEVNULL,
)
async def get_proc_response(proc: asyncio.subprocess.Process, json_decode: bool = True) -> Union[dict, str]:
""" Checks response code and returns response from a Process
"""
stdout, _ = await proc.communicate()
if proc.returncode != 0:
raise RuntimeError(
f"Convoy is not okay: return code {proc.returncode}: {stdout}"
)
if json_decode:
return json.loads(stdout)
return stdout
async def backup_all_volumes():
log_stdout("Checking Convoy...")
info = await get_proc_response(await convoy("info"))
log_stdout(f"Convoy is okay! Getting volumes...", info=info)
volumes = await get_proc_response(await convoy("list"))
# Filter out volumes that are not matching
volumes = list(filter(lambda i: ONLY_MATCHING.findall(i[0]), list(volumes.items())))
for volumes_chunk in chunked_iter(volumes, CONCURRENT_SNAPSHOTS):
tasks = []
for vol_name, vol_data in volumes_chunk:
task = asyncio.create_task(backup_volume(vol_name, vol_data))
tasks.append(task)
await asyncio.wait(tasks)
async def backup_volume(vol_name: str, vol_data: str):
snapshot_name = await perform_volume_snapshot(vol_name, vol_data)
# Remove old snapshots if there are more than KEEP_LAST_SNAPSHOTS
await clean_older_snapshots(vol_data)
# Create backup from snapshot
await perform_snapshot_backup(vol_name, snapshot_name)
# Clean up old backups
await clean_older_backups(vol_name)
async def perform_volume_snapshot(vol_name: str, vol_data: dict):
# Make a new snapshot
snapshot_name = f"{vol_name}-{timestamp_sec()}"
await get_proc_response(await convoy(
"snapshot", "create", vol_name, "--name", snapshot_name,
), json_decode=False)
log_stdout(f"Created new snapshot {snapshot_name} for volume {vol_name}")
return snapshot_name
async def perform_snapshot_backup(volume_name: str, snapshot_name: str):
await get_proc_response(await convoy(
"backup", "create", "--dest", BACKUP_DESTINATION, snapshot_name,
), json_decode=False)
async def clean_older_backups(vol_name: dict):
if KEEP_LAST_BACKUPS == -1:
return
backups = await get_proc_response(await convoy(
"backup", "list", "--volume-name", vol_name, BACKUP_DESTINATION,
))
backups = sorted(backups.values(), key=lambda d: d.get("CreatedTime"))
if len(backups) > KEEP_LAST_BACKUPS:
remove_urls = list(map(
lambda backup: backup["BackupURL"],
backups[:len(backups) - KEEP_LAST_BACKUPS],
))
log_stdout(
f"More backups than {KEEP_LAST_BACKUPS=}, removing "
f"{len(remove_urls)}/{len(backups)} backups: {remove_urls}"
)
for backup_url in remove_urls:
await get_proc_response(await convoy(
"backup", "delete", backup_url,
), json_decode=False)
async def clean_older_snapshots(vol_data: dict):
if KEEP_LAST_SNAPSHOTS == -1:
return
snapshots = sorted(vol_data["Snapshots"].values(), key=lambda d: d.get("CreatedTime"))
if len(snapshots) > KEEP_LAST_SNAPSHOTS:
remove_names = list(map(
lambda snap: snap["Name"],
snapshots[:len(snapshots) - KEEP_LAST_SNAPSHOTS],
))
log_stdout(
f"More snapshots than {KEEP_LAST_SNAPSHOTS=}, removing "
f"{len(remove_names)}/{len(snapshots)} snapshots: {remove_names}"
)
for snapshot_name in remove_names:
await get_proc_response(await convoy(
"snapshot", "delete", snapshot_name,
), json_decode=False)
async def main():
while True:
await backup_all_volumes()
log_stdout(f"Backup task completed! Sleeping {BACKUP_INTERVAL} seconds until next iteration")
await asyncio.sleep(BACKUP_INTERVAL)
asyncio.run(main())

13
settings.yml

@ -1,11 +1,11 @@
---
meta:
name: convoy-vault
name: convoy
target: glow-containers
url: https://concourse.dev.maio.me
team: containers
pipeline: convoy-vault
pipeline: convoy
registry:
repository: containers.dev.maio.me/library
@ -16,11 +16,11 @@ meta:
image:
convoy-vault:
context: resource/
repo: (( concat meta.registry.repository "/convoy-vault" ))
repo: (( concat meta.registry.repository "/convoy" ))
tag: latest
source:
uri: https://glow.dev.maio.me/containers/convoy-vault.git
uri: https://glow.dev.maio.me/containers/convoy.git
branch: master
tasks:
@ -43,8 +43,3 @@ meta:
.: (( inject meta.registry.auth ))
convoy:
version: v0.5.2
vault-init:
repo: "containers.dev.maio.me/pirogoeth/vault-init"
tag: "latest"
auth:
.: (( inject meta.registry.auth ))

Loading…
Cancel
Save