| import numpy as np | |
| import numba | |
| import jax.numpy as jnp | |
| import jax | |
| import time | |
| import torch | |
| import pandas as pd | |
| from functools import partial |
NOTE: HDFS is required for Flink's DistributedCache which distributes Python plans to worker nodes. We use BlueData Hadoop CDH nodes.
Remember to make sure you aren't using env.execute(local=True) in your Python plans!
On the master node:
Install git and other useful things that we like
sudo yum install git bzip2 -y
| """ | |
| Minimal character-level Vanilla RNN model. Written by Andrej Karpathy (@karpathy) | |
| BSD License | |
| """ | |
| import numpy as np | |
| # data I/O | |
| data = open('input.txt', 'r').read() # should be simple plain text file | |
| chars = list(set(data)) | |
| data_size, vocab_size = len(data), len(chars) |
| """ | |
| This is a batched LSTM forward and backward pass | |
| """ | |
| import numpy as np | |
| import code | |
| class LSTM: | |
| @staticmethod | |
| def init(input_size, hidden_size, fancy_forget_bias_init = 3): |
| """ | |
| NMF by coordinate descent, designed for sparse data (without missing values) | |
| """ | |
| # Author: Mathieu Blondel <mathieu@mblondel.org> | |
| # License: BSD 3 clause | |
| import numpy as np | |
| import scipy.sparse as sp | |
| import numba |
| import org.apache.spark.mllib.linalg.distributed.RowMatrix | |
| import org.apache.spark.mllib.linalg._ | |
| import org.apache.spark.{SparkConf, SparkContext} | |
| // To use the latest sparse SVD implementation, please build your spark-assembly after this | |
| // change: https://github.com/apache/spark/pull/1378 | |
| // Input tsv with 3 fields: rowIndex(Long), columnIndex(Long), weight(Double), indices start with 0 | |
| // Assume the number of rows is larger than the number of columns, and the number of columns is | |
| // smaller than Int.MaxValue |
| package com.etsy.scalding.jobs | |
| import com.twitter.scalding._ | |
| class TouchTheVoid(args : Args) extends Job(args) { | |
| // 25M (Long, Double) pairs. | |
| val scores = SequenceFile("stuff", ('id, 'score, 'stuff)) | |
| .project('id, 'score) |
| // Just before switching jobs: | |
| // Add one of these. | |
| // Preferably into the same commit where you do a large merge. | |
| // | |
| // This started as a tweet with a joke of "C++ pro-tip: #define private public", | |
| // and then it quickly escalated into more and more evil suggestions. | |
| // I've tried to capture interesting suggestions here. | |
| // | |
| // Contributors: @r2d2rigo, @joeldevahl, @msinilo, @_Humus_, | |
| // @YuriyODonnell, @rygorous, @cmuratori, @mike_acton, @grumpygiant, |
| import org.apache.commons.math3.linear._ | |
| import com.twitter.algebird.Operators._ | |
| import com.twitter.scalding._ | |
| import cascading.pipe.Pipe | |
| import cascading.pipe.joiner.InnerJoin | |
| import cascading.tuple.Fields | |
| object SVD extends Serializable { |
