feat(geo): switch geocoder from self-host Photon to OpenCage API
CI / test (push) Successful in 7m44s
CI / deploy-ift (push) Successful in 3m57s
CI / e2e-ift (push) Successful in 1m16s
CI / deploy-stage (push) Successful in 2m36s
CI / e2e-stage (push) Successful in 4m12s

logic_geo проксирует OpenCage (suggest/geocode/reverse): env OPENCAGE_API_KEY,
OPENCAGE_URL, OPENCAGE_TIMEOUT_MS, OPENCAGE_COUNTRYCODE, OPENCAGE_THROTTLE_RPS
(1 rps под free-тариф, 0 - выключить). HTTP 402/403/429 апстрима ->
503 geo_unavailable; source=opencage. Сервис photon и volume photon-data
удалены из swarm compose; eunit-фикстуры под формат OpenCage.

Refs EventHub/EventHubBack#77
This commit is contained in:
2026-08-18 23:30:15 +03:00
parent 6b70e1336d
commit 73d43e5fdc
5 changed files with 163 additions and 87 deletions
+3 -4
View File
@@ -43,8 +43,7 @@ REMINDER_LEAD_HOURS=24
# Uploads (avatar/cover) — каталог на volume /app/data (Back#71) # Uploads (avatar/cover) — каталог на volume /app/data (Back#71)
UPLOAD_DIR=/app/data/uploads UPLOAD_DIR=/app/data/uploads
UPLOAD_MAX_BYTES=2097152 UPLOAD_MAX_BYTES=2097152
# Photon (DevOps#16). Пусто = /v1/geo 503. Включить: PHOTON_REPLICAS=1 и PHOTON_URL=http://photon:2322 # OpenCage (DevOps#16). = /v1/geo -> 503 (degrade). Free: 2500/, 1 rps.
PHOTON_URL= OPENCAGE_API_KEY=
PHOTON_TIMEOUT_MS=3000 OPENCAGE_TIMEOUT_MS=3000
PHOTON_REPLICAS=0
+2 -26
View File
@@ -70,8 +70,8 @@ services:
- ADMIN_SUPPORT_PASSWORD=${ADMIN_SUPPORT_PASSWORD} - ADMIN_SUPPORT_PASSWORD=${ADMIN_SUPPORT_PASSWORD}
- CLUSTER_MODE=true - CLUSTER_MODE=true
- DNS_NAME=eventhub-node - DNS_NAME=eventhub-node
- PHOTON_URL=${PHOTON_URL:-} - OPENCAGE_API_KEY=${OPENCAGE_API_KEY:-}
- PHOTON_TIMEOUT_MS=${PHOTON_TIMEOUT_MS:-3000} - OPENCAGE_TIMEOUT_MS=${OPENCAGE_TIMEOUT_MS:-3000}
networks: networks:
eventhub-net: eventhub-net:
aliases: aliases:
@@ -235,29 +235,6 @@ services:
max_attempts: 3 max_attempts: 3
window: 120s window: 120s
# OSM geocoder (komoot Photon). Internal only — do not publish :2322.
# Default replicas 0. Enable: PHOTON_REPLICAS=1 PHOTON_URL=http://photon:2322
photon:
image: ${PHOTON_IMAGE:-rtuszik/photon-docker:latest}
environment:
- REGION=${PHOTON_REGION:-russia}
- UPDATE_STRATEGY=${PHOTON_UPDATE_STRATEGY:-PARALLEL}
volumes:
- photon-data:/photon/data
networks:
eventhub-net:
aliases:
- photon
deploy:
replicas: ${PHOTON_REPLICAS:-0}
resources:
limits:
memory: ${PHOTON_MEMORY_LIMIT:-1536M}
reservations:
memory: 128M
restart_policy:
condition: any
networks: networks:
eventhub-net: eventhub-net:
driver: overlay driver: overlay
@@ -269,4 +246,3 @@ volumes:
grafana-data: grafana-data:
traefik-logs: traefik-logs:
loglynx-data: loglynx-data:
photon-data:
+3 -3
View File
@@ -15,7 +15,7 @@ trails() ->
#{ #{
path => <<"/v1/geo/suggest">>, path => <<"/v1/geo/suggest">>,
method => <<"GET">>, method => <<"GET">>,
description => <<"Address typeahead (Photon). Public + IP rate-limit.">>, description => <<"Address typeahead (OpenCage). Public + IP rate-limit.">>,
tags => [<<"Geo">>], tags => [<<"Geo">>],
parameters => [ parameters => [
#{name => <<"q">>, in => <<"query">>, required => true, schema => #{type => string}}, #{name => <<"q">>, in => <<"query">>, required => true, schema => #{type => string}},
@@ -31,7 +31,7 @@ trails() ->
#{ #{
path => <<"/v1/geo/geocode">>, path => <<"/v1/geo/geocode">>,
method => <<"POST">>, method => <<"POST">>,
description => <<"Forward geocode (Photon). Bearer required.">>, description => <<"Forward geocode (OpenCage). Bearer required.">>,
tags => [<<"Geo">>], tags => [<<"Geo">>],
responses => #{ responses => #{
200 => #{description => <<"OK">>}, 200 => #{description => <<"OK">>},
@@ -43,7 +43,7 @@ trails() ->
#{ #{
path => <<"/v1/geo/reverse">>, path => <<"/v1/geo/reverse">>,
method => <<"POST">>, method => <<"POST">>,
description => <<"Reverse geocode (Photon). Bearer required.">>, description => <<"Reverse geocode (OpenCage). Bearer required.">>,
tags => [<<"Geo">>], tags => [<<"Geo">>],
responses => #{ responses => #{
200 => #{description => <<"OK">>}, 200 => #{description => <<"OK">>},
+122 -34
View File
@@ -1,6 +1,7 @@
%%%------------------------------------------------------------------- %%%-------------------------------------------------------------------
%%% @doc Прокси к self-host Photon: suggest / geocode / reverse. %%% @doc Прокси к OpenCage API: suggest / geocode / reverse.
%%% Кэш и rate-limit в ETS. Без PHOTON_URL — {error, unavailable}. %%% Кэш и rate-limit в ETS. Без OPENCAGE_API_KEY — {error, unavailable}.
%%% Глобальный троттлинг апстрима (free-тариф: 1 rps, 2500/сутки).
%%% @end %%% @end
%%%------------------------------------------------------------------- %%%-------------------------------------------------------------------
-module(logic_geo). -module(logic_geo).
@@ -9,11 +10,13 @@
-define(CACHE, eventhub_geo_cache). -define(CACHE, eventhub_geo_cache).
-define(RL, eventhub_geo_rl). -define(RL, eventhub_geo_rl).
-define(UP, eventhub_geo_upstream).
-define(MIN_Q, 3). -define(MIN_Q, 3).
-define(CACHE_TTL_SEC, 604800). -define(CACHE_TTL_SEC, 604800).
-define(RL_WINDOW_SEC, 60). -define(RL_WINDOW_SEC, 60).
-define(RL_SUGGEST, 40). -define(RL_SUGGEST, 40).
-define(RL_WRITE, 20). -define(RL_WRITE, 20).
-define(DEFAULT_URL, <<"https://api.opencagedata.com/geocode/v1/json">>).
-spec ensure() -> ok. -spec ensure() -> ok.
ensure() -> ensure() ->
@@ -27,6 +30,11 @@ ensure() ->
ets:new(?RL, [named_table, public, set, {write_concurrency, true}]); ets:new(?RL, [named_table, public, set, {write_concurrency, true}]);
_ -> ok _ -> ok
end, end,
case ets:info(?UP) of
undefined ->
ets:new(?UP, [named_table, public, set]);
_ -> ok
end,
ok. ok.
-spec suggest(binary(), binary(), binary()) -> -spec suggest(binary(), binary(), binary()) ->
@@ -44,12 +52,12 @@ suggest(Q, Lang, RlKey) ->
case cache_get(CacheKey) of case cache_get(CacheKey) of
{ok, Hits} -> {ok, Hits}; {ok, Hits} -> {ok, Hits};
miss -> miss ->
case photon_get(<<"/api">>, [{<<"q">>, Qn}, {<<"lang">>, LangN}, case opencage_get(forward_q(Qn, LangN, <<"8">>)) of
{<<"limit">>, <<"8">>}]) of
{ok, Body} -> {ok, Body} ->
Hits = features_to_hits(Body), Hits = results_to_hits(Body),
cache_put(CacheKey, Hits), cache_put(CacheKey, Hits),
{ok, Hits}; {ok, Hits};
{error, rate_limited} -> {error, rate_limited};
{error, _} -> {error, unavailable} {error, _} -> {error, unavailable}
end end
end end
@@ -71,16 +79,16 @@ geocode(Q, Lang, RlKey) ->
case cache_get(CacheKey) of case cache_get(CacheKey) of
{ok, Hit} -> {ok, Hit}; {ok, Hit} -> {ok, Hit};
miss -> miss ->
case photon_get(<<"/api">>, [{<<"q">>, Qn}, {<<"lang">>, LangN}, case opencage_get(forward_q(Qn, LangN, <<"1">>)) of
{<<"limit">>, <<"1">>}]) of
{ok, Body} -> {ok, Body} ->
case features_to_hits(Body) of case results_to_hits(Body) of
[Hit | _] -> [Hit | _] ->
Out = Hit#{<<"source">> => <<"photon">>}, Out = Hit#{<<"source">> => <<"opencage">>},
cache_put(CacheKey, Out), cache_put(CacheKey, Out),
{ok, Out}; {ok, Out};
[] -> {error, not_found} [] -> {error, not_found}
end; end;
{error, rate_limited} -> {error, rate_limited};
{error, _} -> {error, unavailable} {error, _} -> {error, unavailable}
end end
end end
@@ -105,19 +113,21 @@ reverse(Lat, Lon, Lang, RlKey) when is_number(Lat), is_number(Lon) ->
miss -> miss ->
LatB = float_to_bin(LatR), LatB = float_to_bin(LatR),
LonB = float_to_bin(LonR), LonB = float_to_bin(LonR),
case photon_get(<<"/reverse">>, [{<<"lat">>, LatB}, {<<"lon">>, LonB}, Qs = [{<<"q">>, <<LatB/binary, ",", LonB/binary>>},
{<<"lang">>, LangN}]) of {<<"language">>, LangN}],
case opencage_get(Qs) of
{ok, Body} -> {ok, Body} ->
Hit = case features_to_hits(Body) of Hit = case results_to_hits(Body) of
[H | _] -> H#{<<"source">> => <<"photon">>}; [H | _] -> H#{<<"source">> => <<"opencage">>};
[] -> [] ->
#{<<"address">> => fallback_addr(LatR, LonR), #{<<"address">> => fallback_addr(LatR, LonR),
<<"lat">> => LatR, <<"lat">> => LatR,
<<"lon">> => LonR, <<"lon">> => LonR,
<<"source">> => <<"photon">>} <<"source">> => <<"opencage">>}
end, end,
cache_put(CacheKey, Hit), cache_put(CacheKey, Hit),
{ok, Hit}; {ok, Hit};
{error, rate_limited} -> {error, rate_limited};
{error, _} -> {error, unavailable} {error, _} -> {error, unavailable}
end end
end end
@@ -184,15 +194,37 @@ cache_put(Key, Val) ->
ets:insert(?CACHE, {Key, Exp, Val}), ets:insert(?CACHE, {Key, Exp, Val}),
ok. ok.
photon_url() -> %% Forward-запрос: q + language + limit (+ countrycode, если задан).
case os:getenv("PHOTON_URL") of forward_q(Q, LangN, Limit) ->
Qs = [{<<"q">>, Q}, {<<"language">>, LangN}, {<<"limit">>, Limit}],
case countrycode() of
<<>> -> Qs;
CC -> [{<<"countrycode">>, CC} | Qs]
end.
api_key() ->
case os:getenv("OPENCAGE_API_KEY") of
false -> <<>>; false -> <<>>;
"" -> <<>>; "" -> <<>>;
K -> list_to_binary(string:trim(K))
end.
base_url() ->
case os:getenv("OPENCAGE_URL") of
false -> ?DEFAULT_URL;
"" -> ?DEFAULT_URL;
Url -> list_to_binary(string:trim(Url)) Url -> list_to_binary(string:trim(Url))
end. end.
countrycode() ->
case os:getenv("OPENCAGE_COUNTRYCODE") of
false -> <<>>;
"" -> <<>>;
CC -> list_to_binary(string:trim(CC))
end.
timeout_ms() -> timeout_ms() ->
case os:getenv("PHOTON_TIMEOUT_MS") of case os:getenv("OPENCAGE_TIMEOUT_MS") of
false -> 3000; false -> 3000;
"" -> 3000; "" -> 3000;
S -> S ->
@@ -204,14 +236,47 @@ timeout_ms() ->
end end
end. end.
photon_get(Path, Qs) -> %% 0 — троттлинг выключен (платный тариф); по умолчанию 1 rps (free).
case photon_url() of throttle_rps() ->
case os:getenv("OPENCAGE_THROTTLE_RPS") of
false -> 1;
"" -> 1;
S ->
try list_to_integer(S) of
N when N >= 0 -> N;
_ -> 1
catch
_:_ -> 1
end
end.
throttle_allow() ->
case throttle_rps() of
0 -> true;
Rps when Rps > 0 ->
NowMs = erlang:monotonic_time(millisecond),
MinGap = 1000 div Rps,
case ets:lookup(?UP, last) of
[{last, Prev}] when NowMs - Prev < MinGap -> false;
_ ->
ets:insert(?UP, {last, NowMs}),
true
end
end.
opencage_get(Qs) ->
case api_key() of
<<>> -> {error, unavailable}; <<>> -> {error, unavailable};
Base0 -> Key ->
Base = strip_slash(Base0), case throttle_allow() of
Query = uri_string:compose_query(Qs), false -> {error, rate_limited};
Url = binary_to_list(<<Base/binary, Path/binary, $?, Query/binary>>), true ->
http_get(Url) Full = [{<<"key">>, Key}, {<<"no_annotations">>, <<"1">>} | Qs],
Query = uri_string:compose_query(Full),
Url = binary_to_list(<<(strip_slash(base_url()))/binary,
$?, Query/binary>>),
http_get(Url)
end
end. end.
strip_slash(B) -> strip_slash(B) ->
@@ -240,27 +305,50 @@ default_http_get(Url) ->
{error, Reason} {error, Reason}
end. end.
features_to_hits(Body) when is_binary(Body) -> results_to_hits(Body) when is_binary(Body) ->
try jsx:decode(Body, [return_maps]) of try jsx:decode(Body, [return_maps]) of
Map when is_map(Map) -> #{<<"results">> := Results} when is_list(Results) ->
Feats = maps:get(<<"features">>, Map, []), lists:filtermap(fun result_to_hit/1, Results);
lists:filtermap(fun feature_to_hit/1, Feats);
_ -> [] _ -> []
catch catch
_:_ -> [] _:_ -> []
end. end.
feature_to_hit(#{<<"geometry">> := #{<<"coordinates">> := [Lon, Lat | _]}} = F) -> result_to_hit(#{<<"geometry">> := #{<<"lat">> := Lat, <<"lng">> := Lng}} = R) ->
Props = maps:get(<<"properties">>, F, #{}),
Addr = format_address(Props),
{true, #{ {true, #{
<<"address">> => Addr, <<"address">> => address_of(R),
<<"lat">> => to_num(Lat), <<"lat">> => to_num(Lat),
<<"lon">> => to_num(Lon) <<"lon">> => to_num(Lng)
}}; }};
feature_to_hit(_) -> result_to_hit(_) ->
false. false.
address_of(R) ->
case bin(maps:get(<<"formatted">>, R, <<>>)) of
<<>> ->
C = maps:get(<<"components">>, R, #{}),
Name = bin(maps:get(<<"name">>, C, <<>>)),
House = bin(maps:get(<<"house_number">>, C, <<>>)),
Road = bin(maps:get(<<"road">>, C, <<>>)),
StreetLine = case {House, Road} of
{<<>>, Rd} -> Rd;
{H, <<>>} -> H;
{H, Rd} -> <<H/binary, " ", Rd/binary>>
end,
City = first_nonempty([
maps:get(<<"city">>, C, <<>>),
maps:get(<<"town">>, C, <<>>),
maps:get(<<"village">>, C, <<>>)
]),
Country = bin(maps:get(<<"country">>, C, <<>>)),
Parts = [P || P <- [Name, StreetLine, City, Country], P =/= <<>>],
case unique_keep(Parts) of
[] -> <<"">>;
Ps -> join_comma(Ps)
end;
Addr -> Addr
end.
to_num(N) when is_integer(N) -> float(N); to_num(N) when is_integer(N) -> float(N);
to_num(N) when is_float(N) -> N; to_num(N) when is_float(N) -> N;
to_num(_) -> 0.0. to_num(_) -> 0.0.
+32 -19
View File
@@ -4,16 +4,22 @@
setup() -> setup() ->
catch ets:delete(eventhub_geo_cache), catch ets:delete(eventhub_geo_cache),
catch ets:delete(eventhub_geo_rl), catch ets:delete(eventhub_geo_rl),
catch ets:delete(eventhub_geo_upstream),
logic_geo:ensure(), logic_geo:ensure(),
application:unset_env(eventhub, geo_http_get), application:unset_env(eventhub, geo_http_get),
os:putenv("PHOTON_URL", "http://photon.test"), os:putenv("OPENCAGE_API_KEY", "test-key"),
os:putenv("OPENCAGE_URL", "http://opencage.test/geocode/v1/json"),
os:putenv("OPENCAGE_THROTTLE_RPS", "0"),
ok. ok.
cleanup(_) -> cleanup(_) ->
application:unset_env(eventhub, geo_http_get), application:unset_env(eventhub, geo_http_get),
os:unsetenv("PHOTON_URL"), os:unsetenv("OPENCAGE_API_KEY"),
os:unsetenv("OPENCAGE_URL"),
os:unsetenv("OPENCAGE_THROTTLE_RPS"),
catch ets:delete(eventhub_geo_cache), catch ets:delete(eventhub_geo_cache),
catch ets:delete(eventhub_geo_rl), catch ets:delete(eventhub_geo_rl),
catch ets:delete(eventhub_geo_upstream),
ok. ok.
logic_geo_test_() -> logic_geo_test_() ->
@@ -22,22 +28,21 @@ logic_geo_test_() ->
fun cleanup/1, fun cleanup/1,
[ [
{"Suggest too short", fun test_suggest_short/0}, {"Suggest too short", fun test_suggest_short/0},
{"Suggest parses photon geojson", fun test_suggest_ok/0}, {"Suggest parses opencage results", fun test_suggest_ok/0},
{"Geocode empty features is not_found", fun test_geocode_not_found/0}, {"Geocode empty results is not_found", fun test_geocode_not_found/0},
{"Unavailable without photon url", fun test_unavailable/0}, {"Unavailable without opencage key", fun test_unavailable/0},
{"Upstream quota (402) is unavailable", fun test_quota_exceeded/0},
{"Reverse fallback address", fun test_reverse_empty/0}, {"Reverse fallback address", fun test_reverse_empty/0},
{"Format address from properties", fun test_format_address/0} {"Format address from properties", fun test_format_address/0}
]}. ]}.
sample_geojson() -> sample_opencage() ->
<<"{ <<"{
\"features\": [{ \"status\": {\"code\": 200, \"message\": \"OK\"},
\"geometry\": {\"coordinates\": [37.62, 55.75]}, \"results\": [{
\"properties\": { \"formatted\": \"Red Square, Moscow, Russia\",
\"name\": \"Red Square\", \"geometry\": {\"lat\": 55.75, \"lng\": 37.62},
\"city\": \"Moscow\", \"components\": {\"city\": \"Moscow\", \"country\": \"Russia\"}
\"country\": \"Russia\"
}
}] }]
}">>. }">>.
@@ -45,27 +50,35 @@ test_suggest_short() ->
?assertEqual({error, invalid_query}, logic_geo:suggest(<<"ab">>, <<"ru">>, <<"ip">>)). ?assertEqual({error, invalid_query}, logic_geo:suggest(<<"ab">>, <<"ru">>, <<"ip">>)).
test_suggest_ok() -> test_suggest_ok() ->
application:set_env(eventhub, geo_http_get, fun(_) -> {ok, sample_geojson()} end), application:set_env(eventhub, geo_http_get, fun(_) -> {ok, sample_opencage()} end),
{ok, Hits} = logic_geo:suggest(<<"красная площадь">>, <<"ru">>, <<"ip1">>), {ok, Hits} = logic_geo:suggest(<<"красная площадь">>, <<"ru">>, <<"ip1">>),
?assertMatch([#{<<"lat">> := 55.75, <<"lon">> := 37.62}], Hits), ?assertMatch([#{<<"lat">> := 55.75, <<"lon">> := 37.62}], Hits),
[#{<<"address">> := Addr}] = Hits, [#{<<"address">> := Addr}] = Hits,
?assertNotEqual(<<>>, Addr). ?assertEqual(<<"Red Square, Moscow, Russia">>, Addr).
test_geocode_not_found() -> test_geocode_not_found() ->
application:set_env(eventhub, geo_http_get, fun(_) -> {ok, <<"{\"features\":[]}">>} end), application:set_env(eventhub, geo_http_get,
fun(_) -> {ok, <<"{\"status\":{\"code\":200},\"results\":[]}">>} end),
?assertEqual({error, not_found}, ?assertEqual({error, not_found},
logic_geo:geocode(<<"nowherexyz">>, <<"ru">>, <<"u1">>)). logic_geo:geocode(<<"nowherexyz">>, <<"ru">>, <<"u1">>)).
test_unavailable() -> test_unavailable() ->
os:unsetenv("PHOTON_URL"), os:unsetenv("OPENCAGE_API_KEY"),
application:unset_env(eventhub, geo_http_get), application:unset_env(eventhub, geo_http_get),
?assertEqual({error, unavailable}, ?assertEqual({error, unavailable},
logic_geo:suggest(<<"москва центр">>, <<"ru">>, <<"ip2">>)). logic_geo:suggest(<<"москва центр">>, <<"ru">>, <<"ip2">>)).
test_quota_exceeded() ->
application:set_env(eventhub, geo_http_get,
fun(_) -> {error, {http_status, 402}} end),
?assertEqual({error, unavailable},
logic_geo:suggest(<<"москва центр">>, <<"ru">>, <<"ip3">>)).
test_reverse_empty() -> test_reverse_empty() ->
application:set_env(eventhub, geo_http_get, fun(_) -> {ok, <<"{\"features\":[]}">>} end), application:set_env(eventhub, geo_http_get,
fun(_) -> {ok, <<"{\"status\":{\"code\":200},\"results\":[]}">>} end),
{ok, Hit} = logic_geo:reverse(55.75, 37.62, <<"ru">>, <<"u2">>), {ok, Hit} = logic_geo:reverse(55.75, 37.62, <<"ru">>, <<"u2">>),
?assertEqual(<<"photon">>, maps:get(<<"source">>, Hit)), ?assertEqual(<<"opencage">>, maps:get(<<"source">>, Hit)),
?assertEqual(55.75, maps:get(<<"lat">>, Hit)). ?assertEqual(55.75, maps:get(<<"lat">>, Hit)).
test_format_address() -> test_format_address() ->