The repo shows how to log run results in DBT with ClickHouse SQL.
The idea is to use on-run-end post-hook. The code logs detailed results data, including invocation arguments and timestamps.
This improves upon https://gist.github.com/jeremyyeo/064106e480106b49cd337f33a765ef20.
| # in bash; select the appropriate resource id and use the download api via wget | |
| osf_id=cwu4m | |
| wget --content-disposition "https://osf.io/$osf_id/download" |
| # select an appropriate container tag https://catalog.ngc.nvidia.com/orgs/nvidia/containers/pytorch/tags | |
| docker run -it -v ${PWD}:/workspace --name torch_container --gpus all nvcr.io/nvidia/pytorch:21.07-py3 |
| import os | |
| import requests | |
| from functools import partial | |
| API_KEY = os.getenv('API_KEY') # NOTE: can use the limited 'test' key for demo purposes | |
| QUERY_TEMPLATE = 'https://content.guardianapis.com/{section}/{year}/{month}/{day}/{title}?api-key={API_KEY}&show-blocks=all' # NOTE: can try less info, e.g. &show-fields=BodyText | |
| QUERY_FN = partial(QUERY_TEMPLATE.format,API_KEY=API_KEY) | |
| web_url = 'https://www.theguardian.com/business/2014/sep/10/thorntons-60-per-cent-profits-rise-despite-closures' # NOTE: put your desired url | |
| section, year, month, day, title = web_url.split('/')[3:] |
| print_lcp_config_options | |
| struct netdissect_options * TypeKind.POINTER arg declared in ./tcpdump/print-ppp.c:L403,C39-L403,C59 netdissect_options declared in ./tcpdump/netdissect.h:L161 | |
| const unsigned char TypeKind.ELABORATED arg declared in ./tcpdump/print-ppp.c:L403,C61-L403,C73 u_char declared in /Library/Developer/CommandLineTools/SDKs/MacOSX13.sdk/usr/include/sys/_types/_u_char.h:L30 | |
| const unsigned int TypeKind.ELABORATED arg declared in ./tcpdump/print-ppp.c:L403,C75-L403,C86 u_int declared in /Library/Developer/CommandLineTools/SDKs/MacOSX13.sdk/usr/include/sys/_types/_u_int.h:L30 | |
| print_ipcp_config_options | |
| struct netdissect_options * TypeKind.POINTER arg declared in ./tcpdump/print-ppp.c:L404,C40-L404,C60 netdissect_options declared in ./tcpdump/netdissect.h:L161 | |
| const unsigned char * TypeKind.POINTER arg declared in ./tcpdump/print-ppp.c:L404,C62-L404,C77 u_char declared in /Library/Developer/CommandLineTools/SDKs/MacOSX13.sdk/usr/include/sys/_types/_u_char.h:L30 | |
| unsigned int TypeKind.ELABO |
| # gcr.io/kaggle-images/python:latest | |
| # checked on April 21 2023 | |
| # root@9735f873cb2d:/# pip list show | |
| Package Version Editable project location | |
| -------------------------------------- -------------- ------------------------- | |
| absl-py 1.4.0 | |
| accelerate 0.12.0 | |
| access 1.1.9 | |
| affine 2.4.0 | |
| aiobotocore 2.5.0 |
| # Read more about SSH config files: https://linux.die.net/man/5/ssh_config | |
| Host snarto_kaggle_cpu | |
| # update the instance IP under console.cloud.google.com/compute/instances -> more (...) -> network details | |
| HostName 34.170.120.99 | |
| # add the public part under console.cloud.google.com/compute/metadata | |
| IdentityFile ~/.ssh/id_rsa | |
| # match the name under the SSH key | |
| User maciej.skorski | |
| Port 22 |
| # wrap the TF code to be profiled with <start> and <stop> methods as below | |
| tf.profiler.experimental.start('logs/profile') | |
| for (x,y) in data: | |
| train_step(x,y) | |
| tf.profiler.experimental.stop() |
| ## train a logistic regression (images 28x28 and 10 classes) | |
| w = tf.Variable(tf.random.normal(shape=(28*28,10),stddev=0.1),trainable=True) | |
| optimizer = tf.optimizers.SGD(0.01) | |
| @tf.function | |
| def train_step(x, y): | |
| with tf.GradientTape() as tape: | |
| all_logits = tf.matmul(x,w) # (n_batch,n_class) | |
| y_logits = tf.gather(all_logits,y,batch_dims=1) # (n_batch,) |