含山县查询建设工程的网站,织梦网站如何修改数据库配置,网站后台程序和数据库开发,山西p2p网站建设#x1f680; 本文章实现了基于MapReduce的手机浏览日志分析 #x1f680; 文章简介#xff1a;主要包含了数据生成部分#xff0c;数据处理部分#xff0c;数据存储部分与数据可视化部分 #x1f680; 【本文仅供参考#xff01;#xff01;非唯一答案】其中需求实现的… 本文章实现了基于MapReduce的手机浏览日志分析 文章简介主要包含了数据生成部分数据处理部分数据存储部分与数据可视化部分 【本文仅供参考非唯一答案】其中需求实现的方式有多种提供的代码并非唯一写法选择适合的方式即可。 目录 手机日志分析需求数据生成工具类模拟数据生成类MapReduce程序需求编写 手机日志分析需求
本文主要实现以下需求
编写数据生成器生成1G~10G大小的数据字段必须包括id,日期,手机号码、型号、操作系统字段。需要将手机号码4~9为掩码处理。分析2021年、2022年操作系统市场占比、手机型号市场占比情况分析2022年手机运营商市场占比情况分析数据存储到HDFS集群/ana/phone节点下面将分析结果存储到Mysql,并进行数据可视化
数据生成工具类
手机号码随机生成可以采用随机数生成的方式结合三大运营商的号码前三位数为规则进行生成 代码如下
/*** Description 生成三大运营商的手机号*/
/*** 中国移动手机号段* 134、135、136、137、138、139、147、150、151、152、157、158、159、172、178、182、183、184、187、188、198、1703、1705、1706* 中国联通手机号段* 130、131、132、145、155、156、166、171、175、176、185、186、1704、1707、1708、1709* 中国电信手机号段* 133、153、173、177、180、181、189、191、193、199、1700、1701、1702* 腾讯云API https://market.cloud.tencent.com/products/31101*/
public class PhoneNumberGenerator {//生成一万个手机号码只需将 generatePhoneNumbers 方法中的参数 count 修改为 10000 即可//移动private static final String[] CHINA_MOBILE_PREFIX {134, 139, 150, 151, 182};//联通private static final String[] CHINA_UNICOM_PREFIX {130,155,186};//电信private static final String[] CHINA_TELECOM_PREFIX {133,153,180,181,189};public static void main(String[] args) {String phoneNumbers generatePhoneNumbers(1);System.out.println(phoneNumbers);}public static String generatePhoneNumbers(int count) {String phoneNumbernull;Random random new Random();for (int i 0; i count; i) {String prefix;int operatorIndex random.nextInt(3);switch (operatorIndex) {case 0:prefix CHINA_MOBILE_PREFIX[random.nextInt(CHINA_MOBILE_PREFIX.length)];break;case 1:prefix CHINA_UNICOM_PREFIX[random.nextInt(CHINA_UNICOM_PREFIX.length)];break;default:prefix CHINA_TELECOM_PREFIX[random.nextInt(CHINA_TELECOM_PREFIX.length)];}phoneNumber prefix generateRandomNumber(random, 11 - prefix.length());}return replaceCharacters(phoneNumber,3,8);}private static String replaceCharacters(String input, int startIndex, int endIndex) {if (input null || input.length() endIndex) {return input;}StringBuilder sb new StringBuilder(input);for (int i startIndex; i endIndex; i) {sb.setCharAt(i, *);}return sb.toString();}private static String generateRandomNumber(Random random, int length) {StringBuilder sb new StringBuilder();for (int i 0; i length; i) {sb.append(random.nextInt(10));}return sb.toString();}
}运营商解析的其中一种方式 【采用接口分析】这里可以使用鹅厂或者其他厂商开发的接口进行运营商识别 申请获取对应的秘钥即可 例子如下 public class PhoneOperator {public static String calcAuthorization(String source, String secretId, String secretKey, String datetime)throws NoSuchAlgorithmException, UnsupportedEncodingException, InvalidKeyException {String signStr x-date: datetime \n x-source: source;Mac mac Mac.getInstance(HmacSHA1);Key sKey new SecretKeySpec(secretKey.getBytes(UTF-8), mac.getAlgorithm());mac.init(sKey);byte[] hash mac.doFinal(signStr.getBytes(UTF-8));String sig new BASE64Encoder().encode(hash);String auth hmac id\ secretId \, algorithm\hmac-sha1\, headers\x-date x-source\, signature\ sig \;return auth;}public static String urlencode(Map?, ? map) throws UnsupportedEncodingException {StringBuilder sb new StringBuilder();for (Map.Entry?, ? entry : map.entrySet()) {if (sb.length() 0) {sb.append();}sb.append(String.format(%s%s,URLEncoder.encode(entry.getKey().toString(), UTF-8),URLEncoder.encode(entry.getValue().toString(), UTF-8)));}return sb.toString();}public static void main(String[] args) throws NoSuchAlgorithmException, UnsupportedEncodingException, InvalidKeyException {//云市场分配的密钥IdString secretId xx;//云市场分配的密钥KeyString secretKey xx;String source market;Calendar cd Calendar.getInstance();SimpleDateFormat sdf new SimpleDateFormat(EEE, dd MMM yyyy HH:mm:ss GMT, Locale.US);sdf.setTimeZone(TimeZone.getTimeZone(GMT));String datetime sdf.format(cd.getTime());// 签名String auth calcAuthorization(source, secretId, secretKey, datetime);// 请求方法String method POST;// 请求头MapString, String headers new HashMapString, String();headers.put(X-Source, source);headers.put(X-Date, datetime);headers.put(Authorization, auth);// 查询参数MapString, String queryParams new HashMapString, String();queryParams.put(mobile,XXX);// body参数MapString, String bodyParams new HashMapString, String();// url参数拼接String url https://service-8c43o60c-1253285064.gz.apigw.tencentcs.com/release/sms;if (!queryParams.isEmpty()) {url ? urlencode(queryParams);}BufferedReader in null;try {URL realUrl new URL(url);HttpURLConnection conn (HttpURLConnection) realUrl.openConnection();conn.setConnectTimeout(5000);conn.setReadTimeout(5000);conn.setRequestMethod(method);// request headersfor (Map.EntryString, String entry : headers.entrySet()) {conn.setRequestProperty(entry.getKey(), entry.getValue());}// request bodyMapString, Boolean methods new HashMap();methods.put(POST, true);methods.put(PUT, true);methods.put(PATCH, true);Boolean hasBody methods.get(method);if (hasBody ! null) {conn.setRequestProperty(Content-Type, application/x-www-form-urlencoded);conn.setDoOutput(true);DataOutputStream out new DataOutputStream(conn.getOutputStream());out.writeBytes(urlencode(bodyParams));out.flush();out.close();}// 定义 BufferedReader输入流来读取URL的响应in new BufferedReader(new InputStreamReader(conn.getInputStream()));String line;String result ;while ((line in.readLine()) ! null) {result line;}System.out.println(result);} catch (Exception e) {System.out.println(e);e.printStackTrace();} finally {try {if (in ! null) {in.close();}} catch (Exception e2) {e2.printStackTrace();}}}
}结果如下 另一种方式为直接根据前三位手机号进行判断
模拟数据生成类
数据生成器 id,日期,手机号码、型号、操作系统 /*** Description* 数据生成器 id,日期,手机号码、型号、操作系统* id:UUID 随机生成 日期:2021 2022 手机号码:三大运营商 型号:Apple HuaWei Oppo Vivo Meizu Nokia 操作系统:Apple ios Harmony Samsung* 1.分析2021年、2022年操作系统市场占比、手机型号市场占比情况* 2.分析2022年手机运营商市场占比情况* 3.分析数据存储到HDFS集群/ana/phone节点下面* 4.将分析结果存储到Mysql,并进行数据可视化*/
public class DataGenerator {public static void main(String[] args) {try {BufferedWriter writer new BufferedWriter(new FileWriter(data/phone.log));for (int i 0; i 1000; i) {//UUID随机生成 id,日期,手机号码、型号、操作系统String id UUID.randomUUID().toString();String date getRandomDate();String phoneNumber PhoneNumberGenerator.generatePhoneNumbers(1);String model getRandomModel();String operatingSystem getRandomOperatingSystem();String line id , date , phoneNumber , model , operatingSystem;writer.write(line);writer.newLine();}writer.close();} catch (IOException e) {e.printStackTrace();}}private static String getRandomDate() {Random random new Random();int year random.nextInt(2) 0 ? 2021 : 2022;int month random.nextInt(12) 1;int dayOfMonth;if (month 2) {dayOfMonth random.nextInt(28) 1;} else if (month 4 || month 6 || month 9 || month 11) {dayOfMonth random.nextInt(30) 1;} else {dayOfMonth random.nextInt(31) 1;}return year - (month 10 ? 0 : ) month - (dayOfMonth10? 0:)dayOfMonth ;}private static String getRandomPhoneNumber() {Random random new Random();StringBuilder phoneNumber new StringBuilder(1);for (int i 0; i 10; i) {phoneNumber.append(random.nextInt(10));}return phoneNumber.toString();}private static String getRandomModel() {String[] models {Apple, HuaWei, Oppo, Vivo, Meizu, Nokia};return models[new Random().nextInt(models.length)];}private static String getRandomOperatingSystem() {String[] operatingSystems {Apple, HarmonyOS, Samsung,iOS};return operatingSystems[new Random().nextInt(operatingSystems.length)];}
}结果如下
MapReduce程序需求编写
分析2021年、2022年操作系统市场占比、手机型号市场占比情况 /*** Description*/
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Partitioner;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class PhoneOSAnalysis {private static int totalCount;private static int lineCount2021 0;private static int lineCount2022 0;public static class TokenizerMapper extends MapperObject, Text, Text, DoubleWritable {private final static DoubleWritable one new DoubleWritable(1);private Text word new Text();public void map(Object key, Text value, Context context) throws IOException, InterruptedException {String[] fields value.toString().split(,);if (fields.length 5) {// 操作系统市场占比word.set(fields[1].substring(0, 4) -OS- fields[4]);context.write(word, one);// 手机型号市场占比word.set(fields[1].substring(0, 4) -Model- fields[3]);context.write(word, one);}}}public static class MarketShareReducer extends ReducerText, DoubleWritable, Text, DoubleWritable {private DoubleWritable result new DoubleWritable();public void reduce(Text key, IterableDoubleWritable values, Context context)throws IOException, InterruptedException {double sum 0;for (DoubleWritable val : values) {//这里会根据分组的key来计算sumsum val.get();}int yearTotalCount key.toString().contains(2021) ? lineCount2021 : lineCount2022;double percentage sum / yearTotalCount;result.set(percentage);context.write(key, result);}}public static void main(String[] args) throws Exception {Configuration conf new Configuration();FileSystem fs FileSystem.get(conf);Path inputPath new Path(data/phone.log);FSDataInputStream inputStream fs.open(inputPath);try (BufferedReader reader new BufferedReader(new InputStreamReader(fs.open(inputPath)))) {String line;while ((line reader.readLine()) ! null) {if (line.contains(2021)) {lineCount2021;} else if (line.contains(2022)) {lineCount2022;}}}// totalCount Math.max(lineCount2021, lineCount2022);Job job Job.getInstance(conf, market share analysis);job.setJarByClass(PhoneOSAnalysis.class);job.setMapperClass(TokenizerMapper.class);// 设置自定义分区器job.setPartitionerClass(CustomPartitioner.class);job.setNumReduceTasks(2);job.setReducerClass(MarketShareReducer.class);job.setOutputKeyClass(Text.class);job.setOutputValueClass(DoubleWritable.class);Path outputPath new Path(data/result);if (fs.exists(outputPath)) {fs.delete(outputPath, true);}FileInputFormat.addInputPath(job, new Path(data/phone.log));FileOutputFormat.setOutputPath(job, new Path(String.valueOf(outputPath)));// TextInputFormat.addInputPath(job, new Path(hdfs://192.168.192.100:8020/));
// TextInputFormat.outInputPath(job, new Path(hdfs://192.168.192.100:8020/));System.exit(job.waitForCompletion(true) ? 0 : 1);}public static class CustomPartitioner extends PartitionerText, DoubleWritable {Overridepublic int getPartition(Text key, DoubleWritable value, int numPartitions) {// 根据年份进行分区if (key.toString().contains(2021)) {return 0;} else {return 1;}}}
}分析2022年手机运营商市场占比情况 /*** Description*/
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;public class OperatorMR {private static int lineCount2022 0;public static class TokenizerMapper extends MapperObject, Text, Text, DoubleWritable {private final static DoubleWritable one new DoubleWritable(1);private Text word new Text();public void map(Object key, Text value, Context context) throws IOException, InterruptedException {String[] fields value.toString().split(,);if (fields.length 3 fields[1].contains(2022)) {// 手机运营商市场占比word.set(fields[1].substring(0, 4) -Operator- getCarrier(fields[2]));context.write(word, one);}}private String getCarrier(String phoneNumber) {String prefix phoneNumber.substring(0, 3);switch (prefix) {//133,153,180,181,189case 133:case 153:case 180:case 181:case 189:return 电信;//130,155,186case 130:case 155:case 186:return 联通;default:return 移动;}}}public static class MarketShareReducer extends ReducerText, DoubleWritable, Text, DoubleWritable {private DoubleWritable result new DoubleWritable();public void reduce(Text key, IterableDoubleWritable values, Context context)throws IOException, InterruptedException {double sum 0;for (DoubleWritable val : values) {sum val.get();}double percentage sum / lineCount2022;result.set(percentage);context.write(key, result);}}public static void main(String[] args) throws Exception {Configuration conf new Configuration();FileSystem fs FileSystem.get(conf);Path inputPath new Path(data/phone.log);try (BufferedReader reader new BufferedReader(new InputStreamReader(fs.open(inputPath)))) {String line;while ((line reader.readLine()) ! null) {if (line.contains(2022)) {lineCount2022;}}}Job job Job.getInstance(conf, PhoneOperator);job.setJarByClass(OperatorMR.class);job.setMapperClass(TokenizerMapper.class);job.setReducerClass(MarketShareReducer.class);job.setOutputKeyClass(Text.class);job.setOutputValueClass(DoubleWritable.class);Path outputPath new Path(data/result-phone);if (fs.exists(outputPath)) {fs.delete(outputPath, true);}
// TextInputFormat.addInputPath(job, new Path(hdfs://192.168.192.100:8020/));
// TextInputFormat.outInputPath(job, new Path(hdfs://192.168.192.100:8020/));FileInputFormat.addInputPath(job, new Path(data/phone.log));FileOutputFormat.setOutputPath(job, outputPath);System.exit(job.waitForCompletion(true) ? 0 : 1);}
}结果如下
-将分析结果存储到Mysql,并进行数据可视化
package com.yopai.mrmysql;/*** Description*/import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DoubleWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.db.DBConfiguration;
import org.apache.hadoop.mapreduce.lib.db.DBOutputFormat;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;public class OPMysqlMR {private static int lineCount2022 0;public static class TokenizerMapper extends MapperObject, Text, Text, DoubleWritable {private final static DoubleWritable one new DoubleWritable(1);private Text word new Text();public void map(Object key, Text value, Context context) throws IOException, InterruptedException {String[] fields value.toString().split(,);if (fields.length 3 fields[1].contains(2022)) {// 手机运营商市场占比word.set(fields[1].substring(0, 4) -Operator- getCarrier(fields[2]));context.write(word, one);}}private String getCarrier(String phoneNumber) {String prefix phoneNumber.substring(0, 3);switch (prefix) {case 133:case 153:case 180:case 181:case 189:return 电信;case 130:case 155:case 186:return 联通;default:return 移动;}}}public static class MarketShareReducer extends ReducerText, DoubleWritable, DBOutputWritable, NullWritable {private DoubleWritable result new DoubleWritable();public void reduce(Text key, IterableDoubleWritable values, Context context)throws IOException, InterruptedException {double sum 0;for (DoubleWritable val : values) {sum val.get();}double percentage sum / lineCount2022;result.set(percentage);context.write(new DBOutputWritable(key.toString(), result.get()), NullWritable.get());}}public static void main(String[] args) throws Exception {Configuration conf new Configuration();// 设置数据库连接信息String dbUrl jdbc:mysql://localhost:3306/blog;String dbUsername root;String dbPassword Admin2022!;DBConfiguration.configureDB(conf, com.mysql.jdbc.Driver, dbUrl, dbUsername, dbPassword);try (Connection connection DriverManager.getConnection(dbUrl, dbUsername, dbPassword)) {String createTableSql CREATE TABLE IF NOT EXISTS operator_market_share(operator VARCHAR(255), market_share DOUBLE);PreparedStatement preparedStatement connection.prepareStatement(createTableSql);preparedStatement.executeUpdate();}FileSystem fs FileSystem.get(conf);Path inputPath new Path(data/phone.log);try (BufferedReader reader new BufferedReader(new InputStreamReader(fs.open(inputPath)))) {String line;while ((line reader.readLine()) ! null) {if (line.contains(2022)) {lineCount2022;}}}Job job Job.getInstance(conf, PhoneOperator);job.setJarByClass(OPMysqlMR.class);job.setMapperClass(TokenizerMapper.class);job.setMapOutputKeyClass(Text.class);job.setMapOutputValueClass(DoubleWritable.class);job.setReducerClass(MarketShareReducer.class);job.setOutputKeyClass(DBOutputWritable.class);job.setOutputValueClass(NullWritable.class);// 设置数据库输出DBOutputFormat.setOutput(job, operator_market_share, operator, market_share);FileInputFormat.addInputPath(job, new Path(data/phone.log));System.exit(job.waitForCompletion(true) ? 0 : 1);}
}package com.yopai.mrmysql;import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.lib.db.DBWritable;import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;/*** Description*/
public class DBOutputWritable implements Writable, DBWritable {private String operator;private double market_share;public DBOutputWritable() {}public DBOutputWritable(String operator, double market_share) {this.operator operator;this.market_share market_share;}Overridepublic void readFields(DataInput in) throws IOException {operator in.readUTF();market_share in.readDouble();}Overridepublic void write(DataOutput out) throws IOException, IOException {out.writeUTF(operator);out.writeDouble(market_share);}Overridepublic void readFields(ResultSet resultSet) throws SQLException {// 不需要实现此方法因为我们只会写入数据到数据库}Overridepublic void write(PreparedStatement preparedStatement) throws SQLException {preparedStatement.setString(1, operator);preparedStatement.setDouble(2, market_share);}
}运行结果如下
可视化操作
package com.yopai.draw;/*** Description*/
import java.awt.*;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.Statement;
import javax.swing.JFrame;
import org.jfree.chart.ChartFactory;
import org.jfree.chart.ChartPanel;
import org.jfree.chart.JFreeChart;
import org.jfree.chart.plot.PiePlot;
import org.jfree.data.general.DefaultPieDataset;public class PieChartExample extends JFrame {public PieChartExample() {// 从数据库获取数据DefaultPieDataset dataset new DefaultPieDataset();try {String dbUrl jdbc:mysql://localhost:3306/blog;String dbUsername root;String dbPassword Admin2022!;Connection connection DriverManager.getConnection(dbUrl, dbUsername, dbPassword);Statement statement connection.createStatement();ResultSet resultSet statement.executeQuery(SELECT operator, market_share FROM operator_market_share);while (resultSet.next()) {String operator resultSet.getString(operator);double marketShare resultSet.getDouble(market_share);dataset.setValue(operator, marketShare);}} catch (Exception e) {e.printStackTrace();}// 创建饼图JFreeChart pieChart ChartFactory.createPieChart(运营商市场占比, // 图表标题dataset, // 数据集true, // 是否显示图例true, // 是否生成工具提示false // 是否生成URL链接);// 设置字体以显示中文Font font new Font(宋体, Font.PLAIN, 12);pieChart.getTitle().setFont(font);pieChart.getLegend().setItemFont(font);PiePlot plot (PiePlot) pieChart.getPlot();plot.setLabelFont(font);// 添加饼图到面板并显示ChartPanel chartPanel new ChartPanel(pieChart);setContentPane(chartPanel);}public static void main(String[] args) {PieChartExample pieChartExample new PieChartExample();pieChartExample.setSize(600, 600);pieChartExample.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);pieChartExample.setVisible(true);}
}结果如下 本篇文章到这里结束 需要注意的是每个人的环境不用调用的API会有所差异。
