From f0e78c9c4a65e88f6f6427018ce85ac6e01fb8dc Mon Sep 17 00:00:00 2001 From: IP2Location Date: Tue, 28 Jun 2022 15:12:58 +0800 Subject: [PATCH] Reduced file I/O --- LICENSE.TXT | 2 +- ip2location.erl | 310 ++++++++++++++++++++++++------------------------ 2 files changed, 158 insertions(+), 154 deletions(-) diff --git a/LICENSE.TXT b/LICENSE.TXT index a54dedb..ed5c13b 100644 --- a/LICENSE.TXT +++ b/LICENSE.TXT @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2021 IP2Location.com +Copyright (c) 2022 IP2Location.com Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/ip2location.erl b/ip2location.erl index 59f677d..ca37dd4 100644 --- a/ip2location.erl +++ b/ip2location.erl @@ -27,7 +27,7 @@ -define(IF(Cond), (case (Cond) of true -> (0); false -> (1) end)). apiversion() -> - "8.4.0". + "8.5.0". getapiversion() -> io:format("API Version: ~p~n", [apiversion()]). @@ -36,42 +36,27 @@ round(Number, Precision) -> P = math:pow(10, Precision), round(Number * P) / P. -readuint(S, StartPos, Len) -> - case file:pread(S, StartPos - 1, Len) of - eof -> - ok; - {ok, Data} -> - binary:decode_unsigned(Data, little) - end. - readuintrow(R, StartPos, Len) -> Data = binary:part(R, StartPos, Len), binary:decode_unsigned(Data, little). -readuint8(S, StartPos) -> - readuint(S, StartPos, 1). - -readuint32(S, StartPos) -> - readuint(S, StartPos, 4). +readuint8row(R, StartPos) -> + readuintrow(R, StartPos, 1). readuint32row(R, StartPos) -> readuintrow(R, StartPos, 4). -readuint128(S, StartPos) -> - readuint(S, StartPos, 16). +readuint128row(R, StartPos) -> + readuintrow(R, StartPos, 16). readstr(S, StartPos) -> - case file:pread(S, StartPos, 1) of + case file:pread(S, StartPos, 256) of % max size of string field + 1 byte for the length eof -> ok; - {ok, LenRaw} -> - Len = binary:decode_unsigned(LenRaw, little), - case file:pread(S, StartPos + 1, Len) of - eof -> - ok; - {ok, Data} -> - binary_to_list(Data) - end + {ok, R} -> + Len = readuint8row(R, 0), + Data = binary:part(R, 1, Len), + binary_to_list(Data) end. readfloatrow(R, StartPos) -> @@ -90,49 +75,55 @@ input(InputFile) -> new(InputFile) -> S = input(InputFile), - Databasetype = readuint8(S, 1), - Databasecolumn = readuint8(S, 2), - Databaseyear = readuint8(S, 3), - % Databasemonth = readuint8(S, 4), - % Databaseday = readuint8(S, 5), - Ipv4databasecount = readuint32(S, 6), - Ipv4databaseaddr = readuint32(S, 10), - Ipv6databasecount = readuint32(S, 14), - Ipv6databaseaddr = readuint32(S, 18), - Ipv4indexbaseaddr = readuint32(S, 22), - Ipv6indexbaseaddr = readuint32(S, 26), - Productcode = readuint8(S, 30), - Ipv4columnsize = Databasecolumn bsl 2, % 4 bytes each column - Ipv6columnsize = 16 + ((Databasecolumn - 1) bsl 2), % 4 bytes each column, except IPFrom column which is 16 bytes - % Producttype = readuint8(S, 31), - % Filesize = readuint32(S, 32), - file:close(S), - - if - % check if is correct BIN (should be 1 for IP2Location BIN file), also checking for zipped file (PK being the first 2 chars) - (Productcode /= 1 andalso Databaseyear >= 21) orelse (Databasetype == 80 andalso Databasecolumn == 75) -> - io:format("Incorrect IP2Location BIN file format. Please make sure that you are using the latest IP2Location BIN file.~n", []), - halt(); - true -> - case ets:info(mymeta) of - undefined -> - ets:new(mymeta, [set, named_table]), - ets:insert(mymeta, {inputfile, InputFile}), - ets:insert(mymeta, {databasetype, Databasetype}), - ets:insert(mymeta, {databasetype, Databasetype}), - ets:insert(mymeta, {databasecolumn, Databasecolumn}), - ets:insert(mymeta, {ipv4databasecount, Ipv4databasecount}), - ets:insert(mymeta, {ipv4databaseaddr, Ipv4databaseaddr}), - ets:insert(mymeta, {ipv6databasecount, Ipv6databasecount}), - ets:insert(mymeta, {ipv6databaseaddr, Ipv6databaseaddr}), - ets:insert(mymeta, {ipv4indexbaseaddr, Ipv4indexbaseaddr}), - ets:insert(mymeta, {ipv6indexbaseaddr, Ipv6indexbaseaddr}), - ets:insert(mymeta, {ipv4columnsize, Ipv4columnsize}), - ets:insert(mymeta, {ipv6columnsize, Ipv6columnsize}); - _ -> - ok % do nothing - end - end. + case file:pread(S, 0, 64) of % 64-byte header + eof -> + halt(); + {ok, Data} -> + R = Data, + Databasetype = readuint8row(R, 0), + Databasecolumn = readuint8row(R, 1), + Databaseyear = readuint8row(R, 2), + % Databasemonth = readuint8row(R, 3), + % Databaseday = readuint8row(R, 4), + Ipv4databasecount = readuint32row(R, 5), + Ipv4databaseaddr = readuint32row(R, 9), + Ipv6databasecount = readuint32row(R, 13), + Ipv6databaseaddr = readuint32row(R, 17), + Ipv4indexbaseaddr = readuint32row(R, 21), + Ipv6indexbaseaddr = readuint32row(R, 25), + Productcode = readuint8row(R, 29), + Ipv4columnsize = Databasecolumn bsl 2, % 4 bytes each column + Ipv6columnsize = 16 + ((Databasecolumn - 1) bsl 2), % 4 bytes each column, except IPFrom column which is 16 bytes + % Producttype = readuint8row(R, 30), + % Filesize = readuint32row(R, 31), + + if + % check if is correct BIN (should be 1 for IP2Location BIN file), also checking for zipped file (PK being the first 2 chars) + (Productcode /= 1 andalso Databaseyear >= 21) orelse (Databasetype == 80 andalso Databasecolumn == 75) -> + io:format("Incorrect IP2Location BIN file format. Please make sure that you are using the latest IP2Location BIN file.~n", []), + halt(); + true -> + case ets:info(mymeta) of + undefined -> + ets:new(mymeta, [set, named_table]), + ets:insert(mymeta, {inputfile, InputFile}), + ets:insert(mymeta, {databasetype, Databasetype}), + ets:insert(mymeta, {databasetype, Databasetype}), + ets:insert(mymeta, {databasecolumn, Databasecolumn}), + ets:insert(mymeta, {ipv4databasecount, Ipv4databasecount}), + ets:insert(mymeta, {ipv4databaseaddr, Ipv4databaseaddr}), + ets:insert(mymeta, {ipv6databasecount, Ipv6databasecount}), + ets:insert(mymeta, {ipv6databaseaddr, Ipv6databaseaddr}), + ets:insert(mymeta, {ipv4indexbaseaddr, Ipv4indexbaseaddr}), + ets:insert(mymeta, {ipv6indexbaseaddr, Ipv6indexbaseaddr}), + ets:insert(mymeta, {ipv4columnsize, Ipv4columnsize}), + ets:insert(mymeta, {ipv6columnsize, Ipv6columnsize}); + _ -> + ok % do nothing + end + end + end, + file:close(S). readcolcountryrow(S, R, Dbtype, Col) -> X = "This parameter is unavailable for selected data file. Please upgrade the data file.", @@ -180,7 +171,7 @@ readcolfloatstringrow(S, R, Dbtype, Col) -> end end. -readrecord(S, Dbtype, Rowoffset) -> +readrecord(S, R, Dbtype) -> Country_position = [0, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2], Region_position = [0, 0, 0, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3], City_position = [0, 0, 0, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4], @@ -203,94 +194,95 @@ readrecord(S, Dbtype, Rowoffset) -> Addresstype_position = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21], Category_position = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22], - Cols = ?IF(lists:nth(Dbtype, Country_position) == 0) + ?IF(lists:nth(Dbtype, Region_position) == 0) + ?IF(lists:nth(Dbtype, City_position) == 0) + ?IF(lists:nth(Dbtype, Isp_position) == 0) + ?IF(lists:nth(Dbtype, Latitude_position) == 0) + ?IF(lists:nth(Dbtype, Longitude_position) == 0) + ?IF(lists:nth(Dbtype, Domain_position) == 0) + ?IF(lists:nth(Dbtype, Zipcode_position) == 0) + ?IF(lists:nth(Dbtype, Timezone_position) == 0) + ?IF(lists:nth(Dbtype, Netspeed_position) == 0) + ?IF(lists:nth(Dbtype, Iddcode_position) == 0) + ?IF(lists:nth(Dbtype, Areacode_position) == 0) + ?IF(lists:nth(Dbtype, Weatherstationcode_position) == 0) + ?IF(lists:nth(Dbtype, Weatherstationname_position) == 0) + ?IF(lists:nth(Dbtype, Mcc_position) == 0) + ?IF(lists:nth(Dbtype, Mnc_position) == 0) + ?IF(lists:nth(Dbtype, Mobilebrand_position) == 0) + ?IF(lists:nth(Dbtype, Elevation_position) == 0) + ?IF(lists:nth(Dbtype, Usagetype_position) == 0) + ?IF(lists:nth(Dbtype, Addresstype_position) == 0) + ?IF(lists:nth(Dbtype, Category_position) == 0), - Rowlength = Cols bsl 2, + {Country_short, Country_long} = readcolcountryrow(S, R, Dbtype, Country_position), + Region = readcolstringrow(S, R, Dbtype, Region_position), + City = readcolstringrow(S, R, Dbtype, City_position), + Isp = readcolstringrow(S, R, Dbtype, Isp_position), + Latitude = readcolfloatrow(R, Dbtype, Latitude_position), + Longitude = readcolfloatrow(R, Dbtype, Longitude_position), + Domain = readcolstringrow(S, R, Dbtype, Domain_position), + Zipcode = readcolstringrow(S, R, Dbtype, Zipcode_position), + Timezone = readcolstringrow(S, R, Dbtype, Timezone_position), + Netspeed = readcolstringrow(S, R, Dbtype, Netspeed_position), + Iddcode = readcolstringrow(S, R, Dbtype, Iddcode_position), + Areacode = readcolstringrow(S, R, Dbtype, Areacode_position), + Weatherstationcode = readcolstringrow(S, R, Dbtype, Weatherstationcode_position), + Weatherstationname = readcolstringrow(S, R, Dbtype, Weatherstationname_position), + Mcc = readcolstringrow(S, R, Dbtype, Mcc_position), + Mnc = readcolstringrow(S, R, Dbtype, Mnc_position), + Mobilebrand = readcolstringrow(S, R, Dbtype, Mobilebrand_position), + Elevation = readcolfloatstringrow(S, R, Dbtype, Elevation_position), + Usagetype = readcolstringrow(S, R, Dbtype, Usagetype_position), + Addresstype = readcolstringrow(S, R, Dbtype, Addresstype_position), + Category = readcolstringrow(S, R, Dbtype, Category_position), - case file:pread(S, Rowoffset - 1, Rowlength) of - eof -> - #ip2locationrecord{}; - {ok, Data} -> - R = Data, - - {Country_short, Country_long} = readcolcountryrow(S, R, Dbtype, Country_position), - Region = readcolstringrow(S, R, Dbtype, Region_position), - City = readcolstringrow(S, R, Dbtype, City_position), - Isp = readcolstringrow(S, R, Dbtype, Isp_position), - Latitude = readcolfloatrow(R, Dbtype, Latitude_position), - Longitude = readcolfloatrow(R, Dbtype, Longitude_position), - Domain = readcolstringrow(S, R, Dbtype, Domain_position), - Zipcode = readcolstringrow(S, R, Dbtype, Zipcode_position), - Timezone = readcolstringrow(S, R, Dbtype, Timezone_position), - Netspeed = readcolstringrow(S, R, Dbtype, Netspeed_position), - Iddcode = readcolstringrow(S, R, Dbtype, Iddcode_position), - Areacode = readcolstringrow(S, R, Dbtype, Areacode_position), - Weatherstationcode = readcolstringrow(S, R, Dbtype, Weatherstationcode_position), - Weatherstationname = readcolstringrow(S, R, Dbtype, Weatherstationname_position), - Mcc = readcolstringrow(S, R, Dbtype, Mcc_position), - Mnc = readcolstringrow(S, R, Dbtype, Mnc_position), - Mobilebrand = readcolstringrow(S, R, Dbtype, Mobilebrand_position), - Elevation = readcolfloatstringrow(S, R, Dbtype, Elevation_position), - Usagetype = readcolstringrow(S, R, Dbtype, Usagetype_position), - Addresstype = readcolstringrow(S, R, Dbtype, Addresstype_position), - Category = readcolstringrow(S, R, Dbtype, Category_position), - - #ip2locationrecord{ - country_short = Country_short, - country_long = Country_long, - region = Region, - city = City, - isp = Isp, - latitude = Latitude, - longitude = Longitude, - domain = Domain, - zipcode = Zipcode, - timezone = Timezone, - netspeed = Netspeed, - iddcode = Iddcode, - areacode = Areacode, - weatherstationcode = Weatherstationcode, - weatherstationname = Weatherstationname, - mcc = Mcc, - mnc = Mnc, - mobilebrand = Mobilebrand, - elevation = Elevation, - usagetype = Usagetype, - addresstype = Addresstype, - category = Category - } - end. + #ip2locationrecord{ + country_short = Country_short, + country_long = Country_long, + region = Region, + city = City, + isp = Isp, + latitude = Latitude, + longitude = Longitude, + domain = Domain, + zipcode = Zipcode, + timezone = Timezone, + netspeed = Netspeed, + iddcode = Iddcode, + areacode = Areacode, + weatherstationcode = Weatherstationcode, + weatherstationname = Weatherstationname, + mcc = Mcc, + mnc = Mnc, + mobilebrand = Mobilebrand, + elevation = Elevation, + usagetype = Usagetype, + addresstype = Addresstype, + category = Category + }. searchtree(S, Ipnum, Dbtype, Low, High, BaseAddr, Colsize, Iptype) -> if Low =< High -> Mid = ((Low + High) bsr 1), Rowoffset = BaseAddr + (Mid * Colsize), - Rowoffset2 = Rowoffset + Colsize, - if - Iptype == ipv4 -> - Ipfrom = readuint32(S, Rowoffset), - Ipto = readuint32(S, Rowoffset2); - true -> - Ipfrom = readuint128(S, Rowoffset), - Ipto = readuint128(S, Rowoffset2) + case Iptype of + ipv4 -> + Firstcol = 4; % 4 bytes + ipv6 -> + Firstcol = 16 % 16 bytes end, - if - Ipnum >= Ipfrom andalso Ipnum < Ipto -> - if - Iptype == ipv4 -> - readrecord(S, Dbtype + 1, Rowoffset + 4); - true -> - readrecord(S, Dbtype + 1, Rowoffset + 16) - end; - true -> - if - Ipnum < Ipfrom -> - searchtree(S, Ipnum, Dbtype, Low, Mid - 1, BaseAddr, Colsize, Iptype); - true -> - searchtree(S, Ipnum, Dbtype, Mid + 1, High, BaseAddr, Colsize, Iptype) - end + Readlen = Colsize + Firstcol, + case file:pread(S, Rowoffset - 1, Readlen) of % reading IP From + whole row + next IP From + eof -> + io:format("Error: IP address not found.~n", []), + {}; % return empty + {ok, R} -> + if + Iptype == ipv4 -> + Ipfrom = readuint32row(R, 0), + Ipto = readuint32row(R, Colsize); + true -> + Ipfrom = readuint128row(R, 0), + Ipto = readuint128row(R, Colsize) + end, + + if + Ipnum >= Ipfrom andalso Ipnum < Ipto -> + Rowlen = Colsize - Firstcol, + R2 = binary:part(R, Firstcol, Rowlen), + + readrecord(S, R2, Dbtype + 1); + true -> + if + Ipnum < Ipfrom -> + searchtree(S, Ipnum, Dbtype, Low, Mid - 1, BaseAddr, Colsize, Iptype); + true -> + searchtree(S, Ipnum, Dbtype, Mid + 1, High, BaseAddr, Colsize, Iptype) + end + end end; true -> io:format("Error: IP address not found.~n", []), @@ -301,9 +293,15 @@ search4(S, Ipnum, Dbtype, Low, High, Baseaddr, Indexbaseaddr, Colsize) -> if Indexbaseaddr > 0 -> Indexpos = ((Ipnum bsr 16) bsl 3) + Indexbaseaddr, - Low2 = readuint32(S, Indexpos), - High2 = readuint32(S, Indexpos + 4), - searchtree(S, Ipnum, Dbtype, Low2, High2, Baseaddr, Colsize, ipv4); + case file:pread(S, Indexpos - 1, 8) of % 4 bytes for each IP From & IP To + eof -> + io:format("Error: IP address not found.~n", []), + {}; % return empty + {ok, R} -> + Low2 = readuint32row(R, 0), + High2 = readuint32row(R, 4), + searchtree(S, Ipnum, Dbtype, Low2, High2, Baseaddr, Colsize, ipv4) + end; true -> searchtree(S, Ipnum, Dbtype, Low, High, Baseaddr, Colsize, ipv4) end. @@ -312,9 +310,15 @@ search6(S, Ipnum, Dbtype, Low, High, Baseaddr, Indexbaseaddr, Colsize) -> if Indexbaseaddr > 0 -> Indexpos = ((Ipnum bsr 112) bsl 3) + Indexbaseaddr, - Low2 = readuint32(S, Indexpos), - High2 = readuint32(S, Indexpos + 4), - searchtree(S, Ipnum, Dbtype, Low2, High2, Baseaddr, Colsize, ipv6); + case file:pread(S, Indexpos - 1, 8) of % 4 bytes for each IP From & IP To + eof -> + io:format("Error: IP address not found.~n", []), + {}; % return empty + {ok, R} -> + Low2 = readuint32row(R, 0), + High2 = readuint32row(R, 4), + searchtree(S, Ipnum, Dbtype, Low2, High2, Baseaddr, Colsize, ipv6) + end; true -> searchtree(S, Ipnum, Dbtype, Low, High, Baseaddr, Colsize, ipv6) end.