serve-expired: new feature, support cache serve-expired feature

ReadMe.md

@@ -575,6 +575,8 @@ https://github.com/pymumu/smartdns/releases
 |blacklist-ip|黑名单IP地址|无|[ip/subnet]，可重复| blacklist-ip 1.2.3.4/16
 |force-AAAA-SOA|强制AAAA地址返回SOA|no|[yes\|no]|force-AAAA-SOA yes
 |prefetch-domain|域名预先获取功能|no|[yes\|no]|prefetch-domain yes
+|serve-expired|过期缓存服务功能|no|[yes\|no]，开启此功能后，如果有请求时尝试回应TTL为0的过期记录，并并发查询记录，以避免查询等待|serve-expired yes
+|serve-expired-ttl|过期缓存服务最长超时时间|0|秒，0：表示停用超时，> 0表示指定的超时的秒数|serve-expired-ttl 0
 |dualstack-ip-selection|双栈IP优选|no|[yes\|no]|dualstack-ip-selection yes
 |dualstack-ip-selection-threshold|双栈IP优选阈值|30ms|毫秒|dualstack-ip-selection-threshold [0-1000]
 
@@ -673,6 +675,10 @@ https://github.com/pymumu/smartdns/releases
     通过`prefetch-domain yes`来启用域名预先获取功能，提高查询命中率。  
     配合上述ttl超时时间，smartdns将在域名ttl即将超时使，再次发送查询请求，并缓存查询结果供后续使用。频繁访问的域名将会持续缓存。此功能将在空闲时消耗更多的CPU。
 
+    * 过期缓存服务功能  
+    通过`serve-expired`来启用过期缓存服务功能，可提高缓存命中率的同时，降低CPU占用。  
+    此功能会在TTL超时后，将返回TTL=0给客户端，并且同时再次发送查询请求，并缓存新的结果给后续使用。
+
 1. 第二DNS如何自定义更多行为  
     第二DNS可以作为其他DNS服务器的上游，提供更多的查询行为，通过bind配置支持可以绑定多个端口，不同端口可设置不同的标志，实现不同的功能，如
 

ReadMe_en.md

@@ -570,6 +570,8 @@ Note: Merlin firmware is derived from ASUS firmware and can theoretically be use
 |blacklist-ip|ip blacklist|None|[ip/subnet], Repeatable，When the filtering server responds IPs in the IP blacklist, The result will be discarded directly| blacklist-ip 1.2.3.4/16
 |force-AAAA-SOA|force AAAA query return SOA|no|[yes\|no]|force-AAAA-SOA yes
 |prefetch-domain|domain prefetch feature|no|[yes\|no]|prefetch-domain yes
+|serve-expired|Cache serve expired feature|no|[yes\|no], Attempts to serve old responses from cache with a TTL of 0 in the response without waiting for the actual resolution to finish.|serve-expired yes
+|serve-expired-ttl|Cache serve expired limite TTL|0|second，0：disable，> 0  seconds after expiration|serve-expired-ttl 0
 |dualstack-ip-selection|Dualstack ip selection|no|[yes\|no]|dualstack-ip-selection yes
 |dualstack-ip-selection-threshold|Dualstack ip select threadhold|30ms|millisecond|dualstack-ip-selection-threshold [0-1000]
 
@@ -670,6 +672,10 @@ Note: Merlin firmware is derived from ASUS firmware and can theoretically be use
     Enable pre-fetching of domain names with `prefetch-domain yes` to improve query hit rate.
     by default, Smartdns will send domain query request again before cache expire, and cache the result for the next query. Frequently accessed domain names will continue to be cached. This feature will consume more CPU when idle.
 
+    * Cache serve expired feature  
+    Enable cache serve expired feature with `serve-expired yes` to improve the cache hit rate and reduce the CPU consumption.
+    This feature will return TTL = 0 to the client after the TTL timeout, and send a new query request again at the same time, and cache the new results for later query.
+
 1. How does the second DNS customize more behavior?
     The second DNS can be used as the upstream of other DNS servers to provide more query behaviors. Bind configuration support can bind multiple ports. Different ports can be set with different flags to implement different functions, such as
 

etc/smartdns/smartdns.conf

@@ -43,6 +43,14 @@ cache-size 512
 # prefetch-domain [yes|no]
 # prefetch-domain yes
 
+# cache serve expired 
+# serve-expired [yes|no]
+# serve-expired yes
+
+# cache serve expired TTL
+# serve-expired-ttl [num]
+# serve-expired-ttl 0
+
 # List of hosts that supply bogus NX domain results 
 # bogus-nxdomain [ip/subnet]
 

package/luci-compat/files/luci/i18n/smartdns.zh-cn.po

@@ -16,6 +16,15 @@ msgstr "SmartDNS是一个本地高性能DNS服务器，支持返回最快IP，
 msgid "Custom Settings"
 msgstr "自定义设置"
 
+msgid "General Settings"
+msgstr "基本设置"
+
+msgid "Settings"
+msgstr "设置"
+
+msgid "Advanced Settings"
+msgstr "高级设置"
+
 msgid "Generate Coredump"
 msgstr "生成coredump"
 
@@ -73,6 +82,12 @@ msgstr "域名预加载"
 msgid "Enable domain prefetch, accelerate domain response speed."
 msgstr "启用域名预加载，加速域名响应速度。"
 
+msgid "Serve expired"
+msgstr "过期缓存服务"
+
+msgid "Attempts to serve old responses from cache with a TTL of 0 in the response without waiting for the actual resolution to finish."
+msgstr "查询性能优化，有请求时尝试回应TTL为0的过期记录，以避免查询等待。"
+
 msgid "Redirect"
 msgstr "重定向"
 
@@ -276,7 +291,3 @@ msgstr "捐助smartdns项目"
 
 msgid "Donate"
 msgstr "捐助"
-
-
-
-

package/luci-compat/files/luci/model/cbi/smartdns/smartdns.lua

@@ -83,6 +83,15 @@ o.cfgvalue    = function(...)
     return Flag.cfgvalue(...) or "0"
 end
 
+---- Domain Serve expired
+o = s:taboption("settings", Flag, "serve_expired", translate("Serve expired"), 
+	translate("Attempts to serve old responses from cache with a TTL of 0 in the response without waiting for the actual resolution to finish."))
+o.rmempty     = false
+o.default     = o.disabled
+o.cfgvalue    = function(...)
+    return Flag.cfgvalue(...) or "0"
+end
+
 ---- Redirect
 o = s:taboption("settings", ListValue, "redirect", translate("Redirect"), translate("SmartDNS redirect mode"))
 o.placeholder = "none"
@@ -260,13 +269,14 @@ o:value("https", translate("https"))
 o.default     = "udp"
 o.rempty      = false
 
--- Doman addresss
-s = m:section(TypedSection, "smartdns", translate("Domain Address"), 
-	translate("Set Specific domain ip address."))
-s.anonymous = true
+s = m:section(TypedSection, "smartdns", translate("Advanced Settings"), translate("Advanced Settings"));
+s.anonymous = true;
+
+s:tab("domain-address", translate("Domain Address"), translate("Set Specific domain ip address."));
+s:tab("blackip-list", translate("IP Blacklist"), translate("Set Specific ip blacklist."));
 
----- address
-addr = s:option(Value, "address",
+-- Doman addresss
+addr = s:taboption("domain-address", Value, "address",
 	translate(""), 
 	translate("Specify an IP address to return for any host in the given domains, Queries in the domains are never forwarded and always replied to with the specified IP address which may be IPv4 or IPv6."))
 
@@ -283,12 +293,7 @@ function addr.write(self, section, value)
 end
 
 -- IP Blacklist
-s = m:section(TypedSection, "smartdns", translate("IP Blacklist"), 
-	translate("Set Specific ip blacklist."))
-s.anonymous = true
-
----- blacklist
-addr = s:option(Value, "blacklist_ip",
+addr = s:taboption("blackip-list", Value, "blacklist_ip",
 	translate(""), 
 	translate("Configure IP blacklists that will be filtered from the results of specific DNS server."))
 
@@ -304,7 +309,7 @@ function addr.write(self, section, value)
 	nixio.fs.writefile("/etc/smartdns/blacklist-ip.conf", value)
 end
 
--- Doman addresss
+-- Technical Support
 s = m:section(TypedSection, "smartdns", translate("Technical Support"), 
 	translate("If you like this software, please buy me a cup of coffee."))
 s.anonymous = true

package/luci-compat/files/luci/model/cbi/smartdns/upstream.lua

@@ -55,6 +55,21 @@ o:value("https", translate("https"))
 o.default     = "udp"
 o.rempty      = false
 
+---- server group
+o = s:option(Value, "server_group", translate("Server Group"), translate("DNS Server group belongs to, used with nameserver, such as office, home."))
+o.rmempty     = true
+o.placeholder = "default"
+o.datatype    = "hostname"
+o.rempty      = true
+
+---- blacklist_ip
+o = s:option(Flag, "blacklist_ip", translate("IP Blacklist Filtering"), translate("Filtering IP with blacklist"))
+o.rmempty     = false
+o.default     = o.disabled
+o.cfgvalue    = function(...)
+    return Flag.cfgvalue(...) or "0"
+end
+
 ---- TLS host verify
 o = s:option(Value, "tls_host_verify", translate("TLS Hostname Verify"), translate("Set TLS hostname to verify."))
 o.default     = ""
@@ -78,21 +93,6 @@ o.datatype    = "hostname"
 o.rempty      = true
 o:depends("type", "https")
 
----- server group
-o = s:option(Value, "server_group", translate("Server Group"), translate("DNS Server group belongs to, used with nameserver, such as office, home."))
-o.rmempty     = true
-o.placeholder = "default"
-o.datatype    = "hostname"
-o.rempty      = true
-
----- blacklist_ip
-o = s:option(Flag, "blacklist_ip", translate("IP Blacklist Filtering"), translate("Filtering IP with blacklist"))
-o.rmempty     = false
-o.default     = o.disabled
-o.cfgvalue    = function(...)
-    return Flag.cfgvalue(...) or "0"
-end
-
 ---- anti-Answer-Forgery
 -- o = s:option(Flag, "check_edns", translate("Anti Answer Forgery"), translate("Anti answer forgery, if DNS does not work properly after enabling, please turn off this feature"))
 -- o.rmempty     = false
@@ -110,7 +110,6 @@ o.rempty      = true
 o:depends("type", "tls")
 o:depends("type", "https")
 
-
 ---- other args
 o = s:option(Value, "addition_arg", translate("Additional Server Args"), translate("Additional Args for upstream dns servers"))
 o.default     = ""

package/luci/files/luci/htdocs/luci-static/resources/view/smartdns/smartdns.js

@@ -191,9 +191,14 @@ return L.view.extend({
 		o.rmempty = false;
 		o.default = o.disabled;
 
-		// Domain prefetch load ;
 		o = s.taboption("settings", form.Flag, "prefetch_domain", _("Domain prefetch"),
-			_("Enable domain prefetch, accelerate domain response speed."));
+		_("Enable domain prefetch, accelerate domain response speed."));
+		o.rmempty = false;
+		o.default = o.disabled;
+		
+		// Domain Serve expired
+		o = s.taboption("settings", form.Flag, "serve_expired", _("Serve expired"),
+			_("Attempts to serve old responses from cache with a TTL of 0 in the response without waiting for the actual resolution to finish."));
 		o.rmempty = false;
 		o.default = o.disabled;
 

package/luci/files/luci/i18n/smartdns.zh-cn.po

@@ -88,6 +88,12 @@ msgstr "域名预加载"
 msgid "Enable domain prefetch, accelerate domain response speed."
 msgstr "启用域名预加载，加速域名响应速度。"
 
+msgid "Serve expired"
+msgstr "缓存过期服务"
+
+msgid "Attempts to serve old responses from cache with a TTL of 0 in the response without waiting for the actual resolution to finish."
+msgstr "查询性能优化，有请求时尝试回应TTL为0的过期记录，以避免查询等待。"
+
 msgid "Redirect"
 msgstr "重定向"
 
@@ -291,7 +297,3 @@ msgstr "捐助smartdns项目"
 
 msgid "Donate"
 msgstr "捐助"
-
-
-
-

package/openwrt/files/etc/init.d/smartdns

@@ -292,6 +292,8 @@ load_service()
 	config_get prefetch_domain "$section" "prefetch_domain" "0"
 	[ "$prefetch_domain" = "1" ] && conf_append "prefetch-domain" "yes"
 
+	config_get serve_expired "$section" "serve_expired" "0"
+	[ "$serve_expired" = "1" ] && conf_append "serve-expired" "yes"
 
 	SMARTDNS_PORT="$port"
 

src/dns_cache.c

@@ -28,19 +28,25 @@
 struct dns_cache_head {
 	DECLARE_HASHTABLE(cache_hash, 10);
 	struct list_head cache_list;
+	struct list_head inactive_list;
 	atomic_t num;
 	int size;
+	int enable_inactive;
+	int inactive_list_expired;
 	pthread_mutex_t lock;
 };
 
 static struct dns_cache_head dns_cache_head;
 
-int dns_cache_init(int size)
+int dns_cache_init(int size, int enable_inactive, int inactive_list_expired)
 {
 	INIT_LIST_HEAD(&dns_cache_head.cache_list);
+	INIT_LIST_HEAD(&dns_cache_head.inactive_list);
 	hash_init(dns_cache_head.cache_hash);
 	atomic_set(&dns_cache_head.num, 0);
 	dns_cache_head.size = size;
+	dns_cache_head.enable_inactive = enable_inactive;
+	dns_cache_head.inactive_list_expired = inactive_list_expired;
 
 	pthread_mutex_init(&dns_cache_head.lock, NULL);
 
@@ -49,11 +55,25 @@ int dns_cache_init(int size)
 
 static __attribute__((unused)) struct dns_cache *_dns_cache_last(void)
 {
+	struct dns_cache *dns_cache = NULL;
+
+	dns_cache = list_last_entry(&dns_cache_head.inactive_list, struct dns_cache, list);
+	if (dns_cache) {
+		return dns_cache;
+	}
+
 	return list_last_entry(&dns_cache_head.cache_list, struct dns_cache, list);
 }
 
 static struct dns_cache *_dns_cache_first(void)
 {
+	struct dns_cache *dns_cache = NULL;
+
+	dns_cache = list_first_entry_or_null(&dns_cache_head.inactive_list, struct dns_cache, list);
+	if (dns_cache) {
+		return dns_cache;
+	}
+
 	return list_first_entry_or_null(&dns_cache_head.cache_list, struct dns_cache, list);
 }
 
@@ -92,6 +112,12 @@ static void _dns_cache_remove(struct dns_cache *dns_cache)
 	dns_cache_release(dns_cache);
 }
 
+static void _dns_cache_move_inactive(struct dns_cache *dns_cache)
+{
+	list_del_init(&dns_cache->list);
+	list_add_tail(&dns_cache->list, &dns_cache_head.inactive_list);
+}
+
 int dns_cache_replace(char *domain, char *cname, int cname_ttl, int ttl, dns_type_t qtype, unsigned char *addr, int addr_len, int speed)
 {
 	struct dns_cache *dns_cache = NULL;
@@ -136,13 +162,16 @@ int dns_cache_replace(char *domain, char *cname, int cname_ttl, int ttl, dns_typ
 		safe_strncpy(dns_cache->cname, cname, DNS_MAX_CNAME_LEN);
 		dns_cache->cname_ttl = cname_ttl;
 	}
+
+	list_del_init(&dns_cache->list);
+	list_add_tail(&dns_cache->list, &dns_cache_head.cache_list);
 	pthread_mutex_unlock(&dns_cache_head.lock);
 
 	dns_cache_release(dns_cache);
 	return 0;
 errout_unlock:
 	pthread_mutex_unlock(&dns_cache_head.lock);
-// errout:
+	// errout:
 	if (dns_cache) {
 		dns_cache_release(dns_cache);
 	}
@@ -263,7 +292,7 @@ struct dns_cache *dns_cache_lookup(char *domain, dns_type_t qtype)
 
 	if (dns_cache_ret) {
 		/* Return NULL if the cache times out */
-		if (now - dns_cache_ret->insert_time > dns_cache_ret->ttl) {
+		if (dns_cache_head.enable_inactive == 0 && (now - dns_cache_ret->insert_time > dns_cache_ret->ttl)) {
 			_dns_cache_remove(dns_cache_ret);
 			dns_cache_ret = NULL;
 		} else {
@@ -328,6 +357,27 @@ void dns_cache_update(struct dns_cache *dns_cache)
 	pthread_mutex_unlock(&dns_cache_head.lock);
 }
 
+void _dns_cache_remove_expired_ttl(time_t *now)
+{
+	struct dns_cache *dns_cache = NULL;
+	struct dns_cache *tmp;
+	int ttl = 0;
+
+	list_for_each_entry_safe(dns_cache, tmp, &dns_cache_head.inactive_list, list)
+	{
+		ttl = dns_cache->insert_time + dns_cache->ttl - *now;
+		if (ttl > 0) {
+			continue;
+		}
+
+		if (dns_cache_head.inactive_list_expired + ttl > 0) {
+			continue;
+		}
+
+		_dns_cache_remove(dns_cache);
+	}
+}
+
 void dns_cache_invalidate(dns_cache_preinvalid_callback callback, int ttl_pre)
 {
 	struct dns_cache *dns_cache = NULL;
@@ -356,9 +406,18 @@ void dns_cache_invalidate(dns_cache_preinvalid_callback callback, int ttl_pre)
 		}
 
 		if (ttl < 0) {
-			_dns_cache_remove(dns_cache);
+			if (dns_cache_head.enable_inactive) {
+				_dns_cache_move_inactive(dns_cache);
+			} else {
+				_dns_cache_remove(dns_cache);
+			}
 		}
 	}
+
+	if (dns_cache_head.enable_inactive && dns_cache_head.inactive_list_expired != 0) {
+		_dns_cache_remove_expired_ttl(&now);
+	}
+
 	pthread_mutex_unlock(&dns_cache_head.lock);
 
 	list_for_each_entry_safe(dns_cache, tmp, &checklist, check_list)
@@ -376,6 +435,11 @@ void dns_cache_destroy(void)
 	struct dns_cache *dns_cache = NULL;
 	struct dns_cache *tmp;
 	pthread_mutex_lock(&dns_cache_head.lock);
+	list_for_each_entry_safe(dns_cache, tmp, &dns_cache_head.inactive_list, list)
+	{
+		_dns_cache_delete(dns_cache);
+	}
+
 	list_for_each_entry_safe(dns_cache, tmp, &dns_cache_head.cache_list, list)
 	{
 		_dns_cache_delete(dns_cache);

src/dns_cache.h

@@ -55,7 +55,7 @@ struct dns_cache {
 	};
 };
 
-int dns_cache_init(int size);
+int dns_cache_init(int size, int enable_inactive, int inactive_list_expired);
 
 int dns_cache_replace(char *domain, char *cname, int cname_ttl, int ttl, dns_type_t qtype, unsigned char *addr, int addr_len, int speed);
 

src/dns_conf.c

@@ -48,6 +48,8 @@ int dns_conf_tcp_idle_time = 120;
 /* cache */
 int dns_conf_cachesize = DEFAULT_DNS_CACHE_SIZE;
 int dns_conf_prefetch = 0;
+int dns_conf_serve_expired = 0;
+int dns_conf_serve_expired_ttl = 0;
 
 /* upstream servers */
 struct dns_servers dns_conf_servers[DNS_MAX_SERVERS];
@@ -1340,6 +1342,8 @@ static struct config_item _config_item[] = {
 	CONF_INT("tcp-idle-time", &dns_conf_tcp_idle_time, 0, 3600),
 	CONF_INT("cache-size", &dns_conf_cachesize, 0, CONF_INT_MAX),
 	CONF_YESNO("prefetch-domain", &dns_conf_prefetch),
+	CONF_YESNO("serve-expired", &dns_conf_serve_expired),
+	CONF_INT("serve-expired-ttl", &dns_conf_serve_expired_ttl, 0, CONF_INT_MAX),
 	CONF_YESNO("dualstack-ip-selection", &dns_conf_dualstack_ip_selection),
 	CONF_INT("dualstack-ip-selection-threshold", &dns_conf_dualstack_ip_selection_threshold, 0, 1000),
 	CONF_CUSTOM("log-level", _config_log_level, NULL),

src/dns_conf.h

@@ -200,6 +200,8 @@ extern int dns_conf_bind_ip_num;
 extern int dns_conf_tcp_idle_time;
 extern int dns_conf_cachesize;
 extern int dns_conf_prefetch;
+extern int dns_conf_serve_expired;
+extern int dns_conf_serve_expired_ttl;
 extern struct dns_servers dns_conf_servers[DNS_MAX_SERVERS];
 extern int dns_conf_server_num;
 

src/dns_server.c

@@ -205,6 +205,8 @@ static struct dns_server server;
 
 static tlog_log *dns_audit;
 
+static int _dns_server_prefetch_request(char *domain, dns_type_t qtype);
+
 static int _dns_server_forward_request(unsigned char *inpacket, int inpacket_len)
 {
 	tlog(TLOG_DEBUG, "forward request.\n");
@@ -1617,6 +1619,7 @@ static int dns_server_resolve_callback(char *domain, dns_result_type rtype, unsi
 
 		/* Not need to wait check result if only has one ip address */
 		if (ip_num == 1 && request_wait == 1) {
+			request->has_ping_result = 1;
 			_dns_server_request_complete(request);
 			_dns_server_request_remove(request);
 		}
@@ -2009,7 +2012,11 @@ static int _dns_server_process_cache(struct dns_request *request)
 		_dns_reply(request);
 	}
 
-	dns_cache_update(dns_cache);
+	if (dns_cache_get_ttl(dns_cache) == 0) {
+		_dns_server_prefetch_request(request->domain, request->qtype);
+	} else {
+		dns_cache_update(dns_cache);
+	}
 	dns_cache_release(dns_cache);
 
 	if (dns_cache_A) {
@@ -3136,7 +3143,7 @@ int dns_server_init(void)
 		return -1;
 	}
 
-	if (dns_cache_init(dns_conf_cachesize) != 0) {
+	if (dns_cache_init(dns_conf_cachesize, dns_conf_serve_expired, dns_conf_serve_expired_ttl) != 0) {
 		tlog(TLOG_ERROR, "init cache failed.");
 		return -1;
 	}