This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
from pylearn2.datasets.dataset import Dataset | |
from pylearn2.datasets.dense_design_matrix import DenseDesignMatrix | |
from pylearn2.utils.iteration import (SequentialSubsetIterator, | |
FiniteDatasetIterator, | |
resolve_iterator_class) | |
import functools | |
import logging | |
import numpy | |
import warnings |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
class AvroUnpackedSourceTest extends Specification with ScalaCheck { | |
"Running M/R jobs on Avro data" should { | |
"work for unpacked sources" in { | |
val prop = Prop.forAll(MyAvroRecordGenerators.myAvroGen) { (record: MyAvroRecord) => | |
var res: Double = 0.0 | |
val tempFolder = new File(System.getProperty("java.io.tmpdir")) | |
val tempInput = new File(tempFolder, "input") | |
tempInput.mkdirs |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package com.ebay.scoobitest | |
import edu.berkeley.cs.avro.marker._ | |
import edu.berkeley.cs.avro.runtime._ | |
import com.nicta.scoobi.Scoobi._ | |
case class LongRec(var f1: Long) extends AvroRecord | |
case class Cluster(var firstSearchTime: Long) extends AvroRecord |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package cascading.avro.examples; | |
import java.util.Properties; | |
import cascading.flow.Flow; | |
import cascading.flow.FlowDef; | |
import cascading.flow.hadoop.HadoopFlowConnector; | |
import cascading.operation.aggregator.Count; | |
import cascading.operation.regex.RegexFilter; | |
import cascading.operation.regex.RegexSplitGenerator; |