Examples¶
The scripts of the directory examples of the repository, with the
credentials that the AWS CLI finds, of any kind, as
Pyjo.AWS.Credentials.Chain finds them.
Amazon S3¶
A bucket of Amazon S3 as a website, with Pyjo.AWS.Client and the
calls of Pyjo.AWS.Protocol.RestXML, whose requests the actions await in
the event loop of the server, without an SDK of AWS: the content of an object,
as it comes, of a call whose protocol forwards it to the client, index.html
of a folder, or the list of its folders and objects.
uv run python examples/s3.py --profile myprofile --bucket mybucket daemon
"""
__DATA__
@@ folder.html.ep
<!DOCTYPE html>
<html>
<head><title>/<%= prefix %></title></head>
<body>
<h1>/<%= prefix %></h1>
<ul>
% if prefix:
<li><a href="../">../</a></li>
% end
% for name in names:
<li><a href="<%= url_for('object', key=prefix + name) %>"><%= name %></a></li>
% end
</ul>
</body>
</html>
"""
import Pyjo.AWS.Client
import Pyjo.AWS.Config
import Pyjo.AWS.Error
import Pyjo.AWS.Protocol.RestXML
import Pyjo.Exception
import Pyjo.GetOpt
import Pyjo.IOLoop
from Pyjo.Util import die, getopt, url_escape
from Pyjoyment.Lite import *
import os
import threading
# The options before the command, with the names of the AWS CLI, which set the environment variables that the
# application and Pyjo.AWS read, but the profile, which comes over the credentials of the environment, as --profile of
# the AWS CLI does, and AWS_PROFILE does not: python s3.py --profile myprofile --bucket mybucket daemon
opts = getopt(
{
'bucket=s': Pyjo.GetOpt.env('S3_BUCKET'),
'endpoint-url=s': Pyjo.GetOpt.env('AWS_ENDPOINT_URL'),
'profile=s': None,
'region=s': Pyjo.GetOpt.env('AWS_REGION'),
},
pass_through=True,
require_order=True,
)
# The bucket, whose objects the application serves, in the path of the endpoint, as S3 and the servers of its API, as
# MinIO, serve it
bucket = os.environ.get('S3_BUCKET') or die(
'Usage: s3.py --bucket BUCKET [--profile PROFILE] [--region REGION] [--endpoint-url URL] COMMAND [OPTIONS]'
)
root = '/' + url_escape(bucket)
# A client of S3 for each event loop, as the connections of its user agent belong to one, with the region, the endpoint
# and the credentials that the AWS CLI finds: of the environment, of a profile with its keys, SSO, aws login, a role
# or credential_process, of a web identity, of the container of ECS or EKS Pod Identity, or of the instance of EC2
local = threading.local()
def s3():
if not hasattr(local, 's3'):
local.s3 = Pyjo.AWS.Client.new(service='s3', config=Pyjo.AWS.Config.new(profile=opts.profile))
return local.s3
# The client closes its connections once the server stops
@hook('after_server_stop')
def after_server_stop(server, app):
if hasattr(local, 's3'):
local.s3.ua.close()
del local.s3
# The path of a key, percent-encoded but its slashes, as S3 signs it
def path(key):
return root + '/' + url_escape(key, r'^A-Za-z0-9\-._~/')
def list_objects(prefix, **query):
return Pyjo.AWS.Protocol.RestXML.new('GET', root, query={'list-type': 2, 'prefix': prefix, **query})
# The headers of an object that the response has, as S3 gives them
HEADERS = (
'Cache-Control',
'Content-Disposition',
'Content-Encoding',
'Content-Language',
'Content-Length',
'Content-Type',
'ETag',
'Last-Modified',
)
# GetObject, whose content goes to the client as it comes, each chunk once the client has read the one before, so that
# an object of any size is not kept in memory; the error of S3, as NoSuchKey, is its XML, which the call reads
class GetObject(Pyjo.AWS.Protocol.RestXML.object):
def __init__(self, c, key):
super().__init__('GET', path(key))
self.c = c
self.gone = False
def build_tx(self, ua, url):
tx = super().build_tx(ua, url)
tx.res.content.set(auto_upgrade=False, auto_decompress=False).once('body', lambda content: self.forward(tx))
return tx
def forward(self, tx):
if tx.res.code != 200:
return
c, res = self.c, tx.res
for name in HEADERS:
if (value := res.headers.header(name)) is not None:
c.res.headers.header(name, value)
if res.headers.content_length == '0':
return c.rendered()
source = Pyjo.IOLoop.stream(tx.connection)
# The content emits an empty chunk too, of the end of the headers that come before it, as S3 sends them in
# records of TLS of their own, which would end the response
def read(content, chunk):
if not chunk:
return
source.stop()
c.write(chunk, lambda c: source.start())
res.content.unsubscribe('read').on('read', read)
# A client that goes away before the end of the object closes the connection to S3, which reads no more
def finish(c):
if not res.is_finished:
self.gone = True
Pyjo.IOLoop.remove(tx.connection)
c.on('finish', finish)
def parse(self, tx):
return None
async def send_object(c, key):
call = GetObject(c, key)
try:
await s3().call_p(call)
except Pyjo.Exception.object:
if not call.gone:
raise
# The folders and the objects under a prefix, of the pages of ListObjectsV2, as S3 has no folders, only keys with
# slashes, and a folder that the console of S3 makes, an empty object of the prefix, which the list leaves out
async def send_folder(c, prefix):
names, count, token = [], 0, None
while True:
dom = await s3().call_p(list_objects(prefix, delimiter='/', **{'continuation-token': token}))
count += int(dom.at('KeyCount').text)
names += [e.text.removeprefix(prefix) for e in dom.find('CommonPrefixes > Prefix')]
names += [e.text.removeprefix(prefix) for e in dom.find('Contents > Key') if e.text != prefix]
if (next_token := dom.at('NextContinuationToken')) is None:
break
token = next_token.text
if prefix and not count:
return c.reply.not_found()
c.render('folder', prefix=prefix, names=sorted(names))
def not_found(e):
return e.code in ('NoSuchKey', '404')
# An object, "index.html" or the list of a folder, for a key that ends with a slash
@get('/*key', {'key': ''}, 'object')
async def serve(c):
key = c.param('key')
if key and not key.endswith('/'):
try:
return await send_object(c, key)
except Pyjo.AWS.Error.object as e:
if not not_found(e):
raise
# A folder without its slash
dom = await s3().call_p(list_objects(f'{key}/', **{'max-keys': 1}))
if int(dom.at('KeyCount').text):
return c.redirect_to('object', key=f'{key}/')
return c.reply.not_found()
try:
await send_object(c, f'{key}index.html')
except Pyjo.AWS.Error.object as e:
if not not_found(e):
raise
await send_folder(c, key)
app.start()