Last active
August 27, 2020 10:58
-
-
Save willprice76/79950586bcb4f09d45f16f4919fa2c66 to your computer and use it in GitHub Desktop.
Adding scoring to the Spark NLP controller
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
package org.example.sparknlp; | |
import com.johnsnowlabs.nlp.LightPipeline; | |
import com.johnsnowlabs.nlp.SparkNLP; | |
import com.johnsnowlabs.nlp.pretrained.PretrainedPipeline; | |
import org.apache.spark.sql.SparkSession; | |
import org.slf4j.Logger; | |
import org.slf4j.LoggerFactory; | |
import org.springframework.web.bind.annotation.PostMapping; | |
import org.springframework.web.bind.annotation.RequestBody; | |
import org.springframework.web.bind.annotation.RestController; | |
import scala.collection.JavaConversions; | |
import scala.collection.Seq; | |
import scala.collection.immutable.Map; | |
import java.time.Duration; | |
import java.time.Instant; | |
import java.util.Arrays; | |
import java.util.List; | |
import java.util.stream.Collectors; | |
@RestController | |
public class SparkNlpController { | |
private static final Logger LOG = LoggerFactory.getLogger(SparkNlpController.class); | |
private LightPipeline scoringPipeline; | |
private SparkSession spark; | |
public SparkNlpController() { | |
spark = SparkNLP.start(false, false); | |
scoringPipeline = new PretrainedPipeline("analyze_sentiment", "en").lightModel(); | |
} | |
@PostMapping("/sentiment/score") | |
public List<String> score(@RequestBody String[] inputData) { | |
Instant start = Instant.now(); | |
LOG.debug("Analyzing {} rows of text data...", inputData.length); | |
//Run test text through the pipeline | |
Map<String, Seq<String>>[] annotations = scoringPipeline.annotate(inputData); | |
//Mangle the sentiment data out | |
List<String> output = Arrays.stream(annotations).map(annotation -> { | |
List<String> sentiment = JavaConversions.seqAsJavaList(annotation.get("sentiment").getOrElse(null)); | |
return sentiment.get(0); | |
}).collect(Collectors.toList()); | |
LOG.debug("Analysis completed in {} milliseconds", Duration.between(start, Instant.now()).toMillis()); | |
return output; | |
} | |
} |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment