dht.h
/*
* Copyright (C) 2014-2017 Savoir-faire Linux Inc.
* Authors: Adrien Béraud <adrien.beraud@savoirfairelinux.com>
* Simon Désaulniers <simon.desaulniers@savoirfairelinux.com>
* Sébastien Blin <sebastien.blin@savoirfairelinux.com>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
#pragma once
#include "infohash.h"
#include "value.h"
#include "utils.h"
#include "network_engine.h"
#include "scheduler.h"
#include "routing_table.h"
#include "callbacks.h"
#include "dht_interface.h"
#include <string>
#include <array>
#include <vector>
#include <map>
#include <functional>
#include <memory>
#ifdef _WIN32
#include <iso646.h>
#endif
namespace dht {
namespace net {
struct Request;
} /* namespace net */
struct Storage;
struct ValueStorage;
class StorageBucket;
struct Listener;
struct LocalListener;
class OPENDHT_PUBLIC Dht final : public DhtInterface {
public:
Dht();
Dht(const int& s, const int& s6, Config config);
virtual ~Dht();
inline const InfoHash& getNodeId() const { return myid; }
NodeStatus getStatus(sa_family_t af) const;
NodeStatus getStatus() const {
return std::max(getStatus(AF_INET), getStatus(AF_INET6));
}
void shutdown(ShutdownCallback cb);
bool isRunning(sa_family_t af = 0) const;
virtual void registerType(const ValueType& type) {
types.registerType(type);
}
const ValueType& getType(ValueType::Id type_id) const {
return types.getType(type_id);
}
void insertNode(const InfoHash& id, const SockAddr&);
void insertNode(const InfoHash& id, const sockaddr* sa, socklen_t salen) {
insertNode(id, SockAddr(sa, salen));
}
void insertNode(const NodeExport& n) {
insertNode(n.id, SockAddr(n.ss, n.sslen));
}
void pingNode(const sockaddr*, socklen_t, DoneCallbackSimple&& cb={});
time_point periodic(const uint8_t *buf, size_t buflen, const SockAddr&);
time_point periodic(const uint8_t *buf, size_t buflen, const sockaddr* from, socklen_t fromlen) {
return periodic(buf, buflen, SockAddr(from, fromlen));
}
virtual void get(const InfoHash& key, GetCallback cb, DoneCallback donecb={}, Value::Filter&& f={}, Where&& w = {});
virtual void get(const InfoHash& key, GetCallback cb, DoneCallbackSimple donecb={}, Value::Filter&& f={}, Where&& w = {}) {
get(key, cb, bindDoneCb(donecb), std::forward<Value::Filter>(f), std::forward<Where>(w));
}
virtual void get(const InfoHash& key, GetCallbackSimple cb, DoneCallback donecb={}, Value::Filter&& f={}, Where&& w = {}) {
get(key, bindGetCb(cb), donecb, std::forward<Value::Filter>(f), std::forward<Where>(w));
}
virtual void get(const InfoHash& key, GetCallbackSimple cb, DoneCallbackSimple donecb, Value::Filter&& f={}, Where&& w = {}) {
get(key, bindGetCb(cb), bindDoneCb(donecb), std::forward<Value::Filter>(f), std::forward<Where>(w));
}
virtual void query(const InfoHash& key, QueryCallback cb, DoneCallback done_cb = {}, Query&& q = {});
virtual void query(const InfoHash& key, QueryCallback cb, DoneCallbackSimple done_cb = {}, Query&& q = {}) {
query(key, cb, bindDoneCb(done_cb), std::forward<Query>(q));
}
std::vector<Sp<Value>> getLocal(const InfoHash& key, Value::Filter f = Value::AllFilter()) const;
Sp<Value> getLocalById(const InfoHash& key, Value::Id vid) const;
void put(const InfoHash& key,
Sp<Value>,
DoneCallback cb=nullptr,
time_point created=time_point::max(),
bool permanent = false);
void put(const InfoHash& key,
const Sp<Value>& v,
DoneCallbackSimple cb,
time_point created=time_point::max(),
bool permanent = false)
{
put(key, v, bindDoneCb(cb), created, permanent);
}
void put(const InfoHash& key,
Value&& v,
DoneCallback cb=nullptr,
time_point created=time_point::max(),
bool permanent = false)
{
put(key, std::make_shared<Value>(std::move(v)), cb, created, permanent);
}
void put(const InfoHash& key,
Value&& v,
DoneCallbackSimple cb,
time_point created=time_point::max(),
bool permanent = false)
{
put(key, std::forward<Value>(v), bindDoneCb(cb), created, permanent);
}
std::vector<Sp<Value>> getPut(const InfoHash&);
Sp<Value> getPut(const InfoHash&, const Value::Id&);
bool cancelPut(const InfoHash&, const Value::Id&);
virtual size_t listen(const InfoHash&, ValueCallback, Value::Filter={}, Where={});
virtual size_t listen(const InfoHash& key, GetCallback cb, Value::Filter f={}, Where w={}) {
return listen(key, [cb](const std::vector<Sp<Value>>& vals, bool expired){
if (not expired)
return cb(vals);
return true;
}, std::forward<Value::Filter>(f), std::forward<Where>(w));
}
virtual size_t listen(const InfoHash& key, GetCallbackSimple cb, Value::Filter f={}, Where w={}) {
return listen(key, bindGetCb(cb), std::forward<Value::Filter>(f), std::forward<Where>(w));
}
virtual bool cancelListen(const InfoHash&, size_t token);
void connectivityChanged(sa_family_t);
void connectivityChanged() {
reported_addr.clear();
connectivityChanged(AF_INET);
connectivityChanged(AF_INET6);
}
std::vector<NodeExport> exportNodes();
std::vector<ValuesExport> exportValues() const;
void importValues(const std::vector<ValuesExport>&);
NodeStats getNodesStats(sa_family_t af) const;
std::string getStorageLog() const;
std::string getStorageLog(const InfoHash&) const;
std::string getRoutingTablesLog(sa_family_t) const;
std::string getSearchesLog(sa_family_t) const;
std::string getSearchLog(const InfoHash&, sa_family_t af = AF_UNSPEC) const;
void dumpTables() const;
std::vector<unsigned> getNodeMessageStats(bool in = false) {
return network_engine.getNodeMessageStats(in);
}
void setStorageLimit(size_t limit = DEFAULT_STORAGE_LIMIT) {
max_store_size = limit;
}
std::pair<size_t, size_t> getStoreSize() const {
return {total_store_size, total_values};
}
std::vector<SockAddr> getPublicAddress(sa_family_t family = 0);
void pushNotificationReceived(const std::map<std::string, std::string>&) {}
void resubscribe(unsigned) {}
private:
/* When performing a search, we search for up to SEARCH_NODES closest nodes
to the destination, and use the additional ones to backtrack if any of
the target 8 turn out to be dead. */
static constexpr unsigned SEARCH_NODES {14};
/* The number of bad nodes is limited in order to help determine
* presence of connectivity changes. See
* https://github.com/savoirfairelinux/opendht/issues/137 for details.
*
* According to the tables, 25 is a good average value for big networks. If
* the network is small, normal search expiration process will handle the
* situation.
* */
static constexpr unsigned SEARCH_MAX_BAD_NODES {25};
/* Concurrent search nodes requested count */
static constexpr unsigned MAX_REQUESTED_SEARCH_NODES {4};
/* Number of listening nodes */
static constexpr unsigned LISTEN_NODES {4};
/* The maximum number of hashes we're willing to track. */
static constexpr unsigned MAX_HASHES {16384};
/* The maximum number of searches we keep data about. */
static constexpr unsigned MAX_SEARCHES {16384};
static constexpr std::chrono::minutes MAX_STORAGE_MAINTENANCE_EXPIRE_TIME {10};
/* The time after which we consider a search to be expirable. */
static constexpr std::chrono::minutes SEARCH_EXPIRE_TIME {62};
/* Timeout for listen */
static constexpr std::chrono::seconds LISTEN_EXPIRE_TIME {30};
static constexpr std::chrono::seconds REANNOUNCE_MARGIN {10};
static constexpr size_t TOKEN_SIZE {32};
// internal structures
struct SearchNode;
struct Get;
struct Announce;
struct Search;
// prevent copy
Dht(const Dht&) = delete;
Dht& operator=(const Dht&) = delete;
InfoHash myid {};
uint64_t secret {};
uint64_t oldsecret {};
// registred types
TypeStore types;
// are we a bootstrap node ?
// note: Any running node can be used as a bootstrap node.
// Only nodes running only as bootstrap nodes should
// be put in bootstrap mode.
const bool is_bootstrap {false};
const bool maintain_storage {false};
// the stuff
RoutingTable buckets4 {};
RoutingTable buckets6 {};
std::map<InfoHash, Storage> store;
std::map<SockAddr, StorageBucket, SockAddr::ipCmp> store_quota;
size_t total_values {0};
size_t total_store_size {0};
size_t max_store_size {DEFAULT_STORAGE_LIMIT};
using SearchMap = std::map<InfoHash, Sp<Search>>;
SearchMap searches4 {};
SearchMap searches6 {};
uint16_t search_id {0};
// map a global listen token to IPv4, IPv6 specific listen tokens.
// 0 is the invalid token.
std::map<size_t, std::tuple<size_t, size_t, size_t>> listeners {};
size_t listener_token {1};
// timing
Scheduler scheduler;
Sp<Scheduler::Job> nextNodesConfirmation {};
Sp<Scheduler::Job> nextStorageMaintenance {};
net::NetworkEngine network_engine;
unsigned pending_pings4 {0};
unsigned pending_pings6 {0};
using ReportedAddr = std::pair<unsigned, SockAddr>;
std::vector<ReportedAddr> reported_addr;
std::mt19937_64 rd {crypto::getSeededRandomEngine<std::mt19937_64>()};
void rotateSecrets();
Blob makeToken(const SockAddr&, bool old) const;
bool tokenMatch(const Blob& token, const SockAddr&) const;
void reportedAddr(const SockAddr&);
// Storage
void storageAddListener(const InfoHash& id, const Sp<Node>& node, size_t tid, Query&& = {});
bool storageStore(const InfoHash& id, const Sp<Value>& value, time_point created, const SockAddr& sa = {});
void expireStore();
void expireStorage(InfoHash h);
void expireStore(decltype(store)::iterator);
void storageChanged(const InfoHash& id, Storage& st, ValueStorage&, bool newValue);
std::string printStorageLog(const decltype(store)::value_type&) const;
void dataPersistence(InfoHash id);
size_t maintainStorage(decltype(store)::value_type&, bool force=false, DoneCallback donecb=nullptr);
// Buckets
RoutingTable& buckets(sa_family_t af) { return af == AF_INET ? buckets4 : buckets6; }
const RoutingTable& buckets(sa_family_t af) const { return af == AF_INET ? buckets4 : buckets6; }
Bucket* findBucket(const InfoHash& id, sa_family_t af) {
RoutingTable::iterator b;
switch (af) {
case AF_INET:
b = buckets4.findBucket(id);
return b == buckets4.end() ? nullptr : &(*b);
case AF_INET6:
b = buckets6.findBucket(id);
return b == buckets6.end() ? nullptr : &(*b);
default:
return nullptr;
}
}
const Bucket* findBucket(const InfoHash& id, sa_family_t af) const {
return const_cast<Dht*>(this)->findBucket(id, af);
}
void expireBuckets(RoutingTable&);
void sendCachedPing(Bucket& b);
bool bucketMaintenance(RoutingTable&);
void dumpBucket(const Bucket& b, std::ostream& out) const;
// Nodes
void onNewNode(const Sp<Node>& node, int confirm);
const Sp<Node> findNode(const InfoHash& id, sa_family_t af) const;
bool trySearchInsert(const Sp<Node>& node);
// Searches
inline SearchMap& searches(sa_family_t af) { return af == AF_INET ? searches4 : searches6; }
inline const SearchMap& searches(sa_family_t af) const { return af == AF_INET ? searches4 : searches6; }
Sp<Search> search(const InfoHash& id, sa_family_t af, GetCallback = {}, QueryCallback = {}, DoneCallback = {}, Value::Filter = {}, Query q = {});
void announce(const InfoHash& id, sa_family_t af, Sp<Value> value, DoneCallback callback, time_point created=time_point::max(), bool permanent = false);
size_t listenTo(const InfoHash& id, sa_family_t af, ValueCallback cb, Value::Filter f = Value::AllFilter(), const Sp<Query>& q = {});
unsigned refill(Search& sr);
void expireSearches();
void confirmNodes();
void expire();
void searchNodeGetDone(const net::Request& status,
net::RequestAnswer&& answer,
std::weak_ptr<Search> ws,
Sp<Query> query);
void searchNodeGetExpired(const net::Request& status, bool over, std::weak_ptr<Search> ws, Sp<Query> query);
void paginate(std::weak_ptr<Search> ws, Sp<Query> query, SearchNode* n);
SearchNode* searchSendGetValues(Sp<Search> sr, SearchNode *n = nullptr, bool update = true);
void searchSendAnnounceValue(const Sp<Search>& sr);
void searchStep(Sp<Search>);
void searchSynchedNodeListen(const Sp<Search>&, SearchNode&);
void dumpSearch(const Search& sr, std::ostream& out) const;
bool neighbourhoodMaintenance(RoutingTable&);
void onError(Sp<net::Request> node, net::DhtProtocolException e);
/* when our address is reported by a distant peer. */
void onReportedAddr(const InfoHash& id, const SockAddr&);
/* when we receive a ping request */
net::RequestAnswer onPing(Sp<Node> node);
/* when we receive a "find node" request */
net::RequestAnswer onFindNode(Sp<Node> node, const InfoHash& hash, want_t want);
void onFindNodeDone(const Sp<Node>& status,
net::RequestAnswer& a,
Sp<Search> sr);
/* when we receive a "get values" request */
net::RequestAnswer onGetValues(Sp<Node> node,
const InfoHash& hash,
want_t want,
const Query& q);
void onGetValuesDone(const Sp<Node>& status,
net::RequestAnswer& a,
Sp<Search>& sr,
const Sp<Query>& orig_query);
/* when we receive a listen request */
net::RequestAnswer onListen(Sp<Node> node,
const InfoHash& hash,
const Blob& token,
size_t socket_id,
const Query& query);
void onListenDone(const Sp<Node>& status,
net::RequestAnswer& a,
Sp<Search>& sr);
/* when we receive an announce request */
net::RequestAnswer onAnnounce(Sp<Node> node,
const InfoHash& hash,
const Blob& token,
const std::vector<Sp<Value>>& v,
const time_point& created);
net::RequestAnswer onRefresh(Sp<Node> node,
const InfoHash& hash,
const Blob& token,
const Value::Id& vid);
void onAnnounceDone(const Sp<Node>& status,
net::RequestAnswer& a,
Sp<Search>& sr);
};
}
dht::DhtInterface
Definition: dht_interface.h:26
dht::Dht
Definition: dht.h:62
dht::Dht::getLocal
std::vector< Sp< Value > > getLocal(const InfoHash &key, Value::Filter f=Value::AllFilter()) const
dht::Dht::exportNodes
std::vector< NodeExport > exportNodes()
dht::Dht::pushNotificationReceived
void pushNotificationReceived(const std::map< std::string, std::string > &)
Definition: dht.h:300
dht::Dht::Dht
Dht(const int &s, const int &s6, Config config)
dht::Dht::connectivityChanged
void connectivityChanged(sa_family_t)
dht::Dht::put
void put(const InfoHash &key, Sp< Value >, DoneCallback cb=nullptr, time_point created=time_point::max(), bool permanent=false)
dht::Dht::getLocalById
Sp< Value > getLocalById(const InfoHash &key, Value::Id vid) const
dht::Dht::getPut
Sp< Value > getPut(const InfoHash &, const Value::Id &)
dht::Dht::setStorageLimit
void setStorageLimit(size_t limit=DEFAULT_STORAGE_LIMIT)
Definition: dht.h:286
dht::Dht::shutdown
void shutdown(ShutdownCallback cb)
dht::Dht::getNodeId
const InfoHash & getNodeId() const
Definition: dht.h:77
dht::Dht::get
virtual void get(const InfoHash &key, GetCallback cb, DoneCallback donecb={}, Value::Filter &&f={}, Where &&w={})
dht::Dht::getPut
std::vector< Sp< Value > > getPut(const InfoHash &)
dht::Dht::getStoreSize
std::pair< size_t, size_t > getStoreSize() const
Definition: dht.h:294
dht::Dht::query
virtual void query(const InfoHash &key, QueryCallback cb, DoneCallback done_cb={}, Query &&q={})
dht::Dht::isRunning
bool isRunning(sa_family_t af=0) const
dht::Dht::listen
virtual size_t listen(const InfoHash &key, GetCallback cb, Value::Filter f={}, Where w={})
Definition: dht.h:235
dht::Dht::insertNode
void insertNode(const InfoHash &id, const SockAddr &)
dht::Dht::cancelPut
bool cancelPut(const InfoHash &, const Value::Id &)
dht::Dht::getStatus
NodeStatus getStatus(sa_family_t af) const
dht::Dht::listen
virtual size_t listen(const InfoHash &, ValueCallback, Value::Filter={}, Where={})
dht::Hash< HASH_LEN >
dht::SockAddr
Definition: sockaddr.h:55
dht::Value::Filter
Definition: value.h:150
dht
Definition: callbacks.h:34
dht::Blob
std::vector< uint8_t > Blob
Definition: utils.h:114
dht::NodeStatus
NodeStatus
Definition: callbacks.h:41
dht::Config
Definition: callbacks.h:89
dht::NodeExport
Definition: infohash.h:363
dht::NodeStats
Definition: callbacks.h:47
dht::Query
Describes a query destined to another peer.
Definition: value.h:864
dht::ValueType
Definition: value.h:79
dht::Value
Definition: value.h:135
dht::Where
Serializable dht::Value filter.
Definition: value.h:740