x MapReduce A Italic URL Constant width Constant width bold Constant width italic 這個圖示代表提示或建議 這個圖示代表一般註解

Transcription

1 Java R Python Java Java Java

2 x MapReduce A Italic URL Constant width Constant width bold Constant width italic 這個圖示代表提示或建議 這個圖示代表一般註解

3 第一章 I/O double[][] int[] String[] List

4 2 Map JavaScript Object Notation JSON somefile.txt id year city id year city int[] id = new int[1024]; int[] year = new int[1024]; String[] city = new String[1024]; BufferedReader clean data int[] row1 = {1, 2014, 1; int[] row2 = {2, 2015, 1; int[] row3 = {3, 2014, 2; int[][] data = {{1, 2014, 1, {2, 2015, 1, {3, 2014, 2;

5 I/O 3 List Map somefile.txt class Record { int id; int year; String city; List Map Record RecordUtils List Record List<Record> listofrecords = new ArrayList<>(); BufferReader Record Record List<Record> listofrecords Record Map Map<String, Record> mapofrecords = new HashMap<>(); ID URL m n X i j m n x i,j x 1, 1 x 1, 2 x 1, n x 2, 1 x 2, 2 x 2, n x m,1 x m,2 x m, n

6 4 JSON JavaScript Object Notation JSON JSON json.org JSON HashMap {"city":"san Francisco", "year": 2020, "id": 2, "event_codes":[20, 22, 34, 19] JSON JSON ArrayList [40, 50, 70, "text", {"city":"san Francisco"] JSON JSON JSON JSON JSON Java null null "null" "NULL" "na" private boolean checknull(string value) { return value == null "null".equalsignorecase(value);

7 I/O String.isEmpty() String.trim() String.isEmpty() true private boolean checkblank(string value) { return value.trim().isempty(); double int long Double Integer Long NumberFormatException try { double d = Double.parseDouble(value); // d catch (NumberFormatException e) { // OffsetDateTime.parse() DateTimeParseException try { OffsetDateTime odt = OffsetDateTime.parse(value); // odt catch (DateTimeParseException e) { //

8 minvalue = 1.0 minvalueinclusive = true 1.0 minvalueinclusive = false 1.0 public boolean checkrange(double value) { boolean minbit = (minvalueinclusive)? value >= minvalue : value > minvalue; boolean maxbit = (maxvalueinclusive)? value <= maxvalue : value < maxvalue; return minbit && maxbit; validitems Set Set.contains() private boolean checkrange(string value) { return validitems.contains(value); DateTime OffsetDateTime OffsetDateTime.isBefore() OffsetDateTime.isAfter() private boolean checkrange(offsetdatetime odt) { return odt.isafter(mindate) && odt.isbefore(maxdate); FileReader String String BufferedReader I/O 1000

9 I/O 7 String[] ASCII ASCII Java Unix bash less somefile.txt bash$ less somefile.txt "id","year","city" 1,2015,"San Francisco" 2,2014,"New York" 3,2012,"Los Angeles"... CSV id year city bash$ wc -l somefile.txt tab TSV JSON 100 bash$ head -100 filename > new_filename 1000

10 8 BufferedReader FileReader FileReader String FileReader File File BufferedReader try(bufferedreader br = new BufferedReader(new FileReader("somefile.txt")) ) { String columnnames = br.readline(); // String line; while ((line = br.readline())!= null) { /* */ // TODO catch (Exception e) { System.err.println(e.getMessage()); // URL url = new URL(" try(bufferedreader br = new BufferedReader( new InputStreamReader(url.openStream())) ) { String columnnames = br.readline(); // String line; while ((line = br.readline())!= null) { // TODO catch (Exception e) { System.err.println(e.getMessage()); // 解析大字串

11 I/O 9 id year city New York = 01, Los Angeles = 02, and San Francisco = 03 String.substring(int beginindex, int endindex) beginindex enindex /* */ int id = Integer.parseInt(line.substring(0, 4)); int year = Integer.parseInt(line.substring(4, 8)); int city = Integer.parseInt(line.substring(8, 10)); 解析分隔字串 CSV CSV 1,2015,"San Francisco" 2,2014,"New York" 3,2012,"Los Angeles" String.split(",") String.trim() String.replace("\", "") /* */ String[] s = line.split(","); int id = Integer.parseInt(s[0].trim()); int year = Integer.parseInt(s[1].trim()); String city = s[2].trim().replace("\"", ""); somefile.txt tab "San Francisco" "New York" "Los Angeles" tab String.split(",") String[] s = line.split("\t");

12 10 CSV - San Francisco, CA regex Apache Commons CSV /* */ CSVParser parser = CSVParser.parse(line, CSVFormat.RFC4180); for(csvrecord cr : parser) { int id = cr.get(1); // 1 0 int year = cr.get(2); String city = cr.get(3); Apache Commons CSV CSVFormat.EXCEL CSVFormat.MYSQL CSVFormat.TDF 解析 JSON 字串 JSON JavaScript API RESTful MongoDB CouchDB NoSQL PostgreSQL 9.3 JSON JSON Java JSON HashMaps ArrayLists - JSON 非單引號 {"id":1, "year":2015, "city":"san Francisco" {"id":2, "year":2014, "city":"new York" {"id":3, "year":2012, "city":"los Angeles" JSON JSON JSON JSON JSON String box org. simple.json /* while JSON */ JSONParser parser = new JSONParser();... /* */

13 I/O 11 JSONObject obj = (JSONObject) parser.parse(line); int id = Integer.parseInt(j.get("id").toString()); int year = Integer.parseInt(j.get("year").toString()); String city = j.get("city").tostring(); JSON JSON JSON ls {{"id":1, "year":2015, "city":"san Francisco", {"id":2, "year":2014, "city":"new York", {"id":3, "year":2012, "city":"los Angeles" Simple JSON JSONParser parser = new JSONParser(); try{ JSONObject jobj = (JSONObject) parser.parse(new FileReader("data.json")); // TODO jobj catch (IOException ParseException e) { System.err.println(e.getMessage()); [{"id":1, "year":2015, "city":"san Francisco", {"id":2, "year":2014, "city":"new York", {"id":3, "year":2012, "city":"los Angeles"] JSON JSONParser parser = new JSONParser(); try{ JSONArray jarr = (JSONArray) parser.parse(new FileReader("data.json")); // TODO jobj catch (IOException ParseException e) { System.err.println(e.getMessage()); 若檔案是每一行一個 JSON 物件, 則檔案技術上並非合法的 JSON 資料結構 讀取檔案與逐行解析 JSON 物件見 讀取文字檔案 一節

14 12 PNG {x 1, x 2 {y D data BufferedImage img = null; try { img = ImageIO.read(new File("Image.png")); int height = img.getheight(); int width = img.getwidth(); int[][] data = new int[height][width]; for (int i = 0; i < height; i++) { for (int j = 0; j < width; j++) { int rgb = img.getrgb(i, j); // data[i][j] = rgb; catch (IOException e) { // RGB int blue = 0x0000ff & rgb; int green = 0x0000ff & (rgb >> 8); int red = 0x0000ff & (rgb >> 16); int alpha = 0x0000ff & (rgb >> 24); byte[] pixels = ((DataBufferByte) img.getraster().getdatabuffer()).getdata(); for (int i = 0; i < pixels.length / 3 ; i++) { int blue = Byte.toUnsignedInt(pixels[3*i]); int green = Byte.toUnsignedInt(pixels[3*i+1]); int red = Byte.toUnsignedInt(pixels[3*i+2]); // rgb (0 1) double gray = ( * red * green * blue) / 255.0

15 I/O 13 2D x n = x 1, x 2,... n m p MNIST A 1D 2D MNIST FileWriter BufferedWriter I/O \t /* */ String output = Integer.toString(record.id) + "," + Integer.toString(record.year) + "," + record.city; Java 8 String.join(delimiter, elements) /* in Java 8 */ String newstring = String.join(",", {"a", "b", "c"); /* Iterator */ String newstring = String.join(",", mylist); Apache Commons Lang StringUtils.join(elements, delimiter) StringBuilder /* in Java 7 */ String[] strings = {"a", "b", "c"; /* StringBuilder */ StringBuilder sb; sb.append(strings[0]); /* */ for(int i = 1; i < strings.length, i++){ /*... \t tab */ sb.append(","); sb.append(strings[i]); String newstring = sb.tostring();

16 14 mystring += mystring_part StringBuilder StringBuilder BufferedWriter.write(String) BufferedWriter.newLine() try(bufferedwriter bw = new BufferedWriter(new FileWriter("somefile.txt")) ) { for(string s : mystringlist){ bw.write(s); /* */ bw.newline(); catch (Exception e) { System.out.println(e.getMessage()); FileWriter false append /* FileWriter append */ try(bufferedwriter bw = new BufferedWriter( new FileWriter("somefile.txt", true))) { for(string s : mystringlist){ bw.write(s); /* */ bw.newline(); catch (Exception e) { System.out.println(e.getMessage()); BufferedWriter PrintWriter PrintWriter println() \n OS PrintWriter try(printwriter pw = new PrintWriter(new BufferedWriter( new FileWriter("somefile.txt"))) ) { for(string s : mystringlist){ /* */ pw.println(s); catch (Exception e) { System.out.println(e.getMessage());

17 I/O 15 JSON JSONObject.toSTring() JSON String String JSON JSONObject obj =... try(bufferedwriter bw = new BufferedWriter(new FileWriter("somefile.txt")) ) { bw.write(obj.tostring()); catch (Exception e) { System.out.println(e.getMessage()); JSON JSON JSONObject List<JSONObject> datalist =... try(bufferedwriter bw = new BufferedWriter(new FileWriter("somefile.txt")) ) { for(jsonobject obj : datalist){ bw.write(obj.tostring()); /* */ bw.newline(); catch (Exception e) { System.out.println(e.getMessage()); FileWriter append FileWriter append JSON try(bufferedwriter bw = new BufferedWriter( new FileWriter("somefile.txt", true)) ) {... MySQL SQL Java Database Connectivity JDBC

18 16 SQL Java MySQL PostgreSQL SQLite MySQL bash$ mysql <database> bash$ mysql -u root <database> bash$ mysql -h host -P port -u user -p password <database> MySQL mysql> SHOW DATABASES; USE dbname mysql> USE mydb; mysql> CREATE TABLE my_table(id INT PRIMARY KEY, stuff VARCHAR(256)); mysql> SOURCE <filename>; mysql> SHOW TABLES; mysql> DESCRIBE <tablename>;

19 I/O 17 SQL ORM SQL 建構 SQL CREATE DATABASE <databasename>; CREATE TABLE <tablename> ( col1 type, col2 type,...); 選取 SELECT SELECT [DISTINCT] col_name, col_name,... col_name FROM table_name [WHERE where_condition] [GROUP BY col_name [ASC DESC]] [HAVING where_condition] [ORDER BY col_name [ASC DESC]] [LIMIT row_count OFFSET offset] [INTO OUTFILE 'file_name'] ORDER BY ORDER BY RAND(); LIMIT ORDER BY RAND() LIMIT 1000; 新增 INSERT INTO tablename(col1, col2,...) VALUES(val1, val2,...);

20 18 INSERT INTO tablename VALUES(val1, val2,...); INSERT INTO tablename(col1, col2,...) VALUES(val1, val2,...),(val1, val2,...), (val1, val2,...); 修改 DBA UPDATE table_name SET col_name = 'value' WHERE other_col_name = 'other_val'; 刪除 UPDATE user_ id record_id DELETE FROM <tablename> WHERE <col_name> = 'col_value'; TRUNCATE TRUNCATE TRUNCATE <tablename>; 拋棄 DROP DROP TABLE <tablename>; DROP DATABASE <databasename>;