This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
<dependency> | |
<groupId>org.apache.spark</groupId> | |
<artifactId>spark-sql_2.11</artifactId> | |
<version>${spark.version}</version> | |
<exclusions> | |
<exclusion> | |
<groupId>org.slf4j</groupId> | |
<artifactId>slf4j-log4j12</artifactId> | |
</exclusion> | |
</exclusions> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
curl -X POST -H --data | |
"[{\"text\": \"I love Spark NLP\", \"label\": \"positive\"}, {\"text\": \"I hate using Scala objects in Java\", \"label\": \"negative\"}]" | |
http://localhost:8080/sentiment/train |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
@PostMapping("/sentiment/train") | |
public String train(@RequestBody List<TextData> data) throws IOException { | |
Instant start = Instant.now(); | |
Dataset<Row> input = spark.createDataFrame(data, TextData.class); | |
LOG.debug("Running training with {} rows of text data", data.size()); | |
Pipeline pipeline = getSentimentTrainingPipeline(); | |
PipelineModel newPipelineModel = pipeline.fit(input); | |
long trainingTime = Duration.between(start, Instant.now()).toMillis(); | |
//Overwrite the existing scoring pipeline | |
scoringPipeline = new LightPipeline(newPipelineModel, false); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package org.example.sparknlp; | |
public class TextData { | |
private String text; | |
private String label; | |
public TextData(String text, String label) { | |
this.text = text; | |
this.label = label; | |
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
private Pipeline getSentimentTrainingPipeline() { | |
DocumentAssembler document = new DocumentAssembler(); | |
document.setInputCol("text"); | |
document.setOutputCol("document"); | |
String[] tokenizerInputCols = {"document"}; | |
Tokenizer tokenizer = new Tokenizer(); | |
tokenizer.setInputCols(tokenizerInputCols); | |
tokenizer.setOutputCol("token"); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
curl -X POST -H "Content-Type: application/json" | |
--data "[\"I love Spark NLP\",\"I hate using Scala objects in Java\"]" | |
http://localhost:8080/sentiment/score |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package org.example.sparknlp; | |
import com.johnsnowlabs.nlp.LightPipeline; | |
import com.johnsnowlabs.nlp.SparkNLP; | |
import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline; | |
import org.apache.spark.sql.SparkSession; | |
import org.slf4j.Logger; | |
import org.slf4j.LoggerFactory; | |
import org.springframework.web.bind.annotation.PostMapping; | |
import org.springframework.web.bind.annotation.RequestBody; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package org.example.sparknlp; | |
import org.apache.spark.sql.SparkSession; | |
import com.johnsnowlabs.nlp.LightPipeline; | |
import com.johnsnowlabs.nlp.SparkNLP; | |
import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline; | |
import com.sun.org.slf4j.internal.Logger; | |
import com.sun.org.slf4j.internal.LoggerFactory; | |
import org.springframework.web.bind.annotation.GetMapping; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package org.example.sparknlp; | |
import org.springframework.web.bind.annotation.GetMapping; | |
import org.springframework.web.bind.annotation.RestController; | |
@RestController | |
public class SparkNlpController { | |
@GetMapping("/hello") | |
public String hello() { | |
return "Hello world"; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package org.example.sparknlp; | |
import org.springframework.web.bind.annotation.GetMapping; | |
import org.springframework.web.bind.annotation.RestController; | |
@RestController | |
public class ExampleController { | |
@GetMapping("/hello") | |
public String hello() { | |
return "Hello world"; |
NewerOlder