Last active
August 29, 2015 14:16
-
-
Save Blackmist/44a0dd871016f4820891 to your computer and use it in GitHub Desktop.
A Hive UDF in C# (for HDInsight)
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| add file wasb:///HiveUDF.exe; | |
| SELECT TRANSFORM (clientid, devicemake, devicemodel) | |
| USING 'HiveUDF.exe' AS | |
| (clientid string, phoneLabel string, phoneHash string) | |
| FROM hivesampletable | |
| ORDER BY clientid LIMIT 50; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| using System; | |
| using System.Security.Cryptography; | |
| using System.Text; | |
| using System.Threading.Tasks; | |
| namespace HiveUDF | |
| { | |
| class Program | |
| { | |
| static void Main(string[] args) | |
| { | |
| string line; | |
| try | |
| { | |
| while ((line = Console.ReadLine()) != null) | |
| { | |
| //do stuff | |
| line = line.TrimEnd('\n'); | |
| string[] field = line.Split('\t'); | |
| string phoneLabel = field[1] + ' ' + field[2]; | |
| Console.WriteLine("{0}\t{1}\t{2}", field[0], phoneLabel, GetMD5Hash(phoneLabel)); | |
| } | |
| } | |
| catch (Exception ex) | |
| { | |
| //bad format or end of line so do nothing | |
| } | |
| } | |
| static string GetMD5Hash(string input) | |
| { | |
| // step 1, calculate MD5 hash from input | |
| MD5 md5 = System.Security.Cryptography.MD5.Create(); | |
| byte[] inputBytes = System.Text.Encoding.ASCII.GetBytes(input); | |
| byte[] hash = md5.ComputeHash(inputBytes); | |
| // step 2, convert byte array to hex string | |
| StringBuilder sb = new StringBuilder(); | |
| for (int i = 0; i < hash.Length; i++) | |
| { | |
| sb.Append(hash[i].ToString("x2")); | |
| } | |
| return sb.ToString(); | |
| } | |
| } | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
A basic example of a C# UDF that can be used from Hive on Azure HDInsight. Note that this works either with Windows-based HDInsight (compiled using VS 2013,) or Linux-based HDInsight (with Mono installed.)