Merge remote-tracking branch 'upstream/master' into multiple_relays

Author: junkurihara

.github/workflows/releases.yml

@@ -28,7 +28,7 @@ jobs:
         run: echo "VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_OUTPUT
 
       - name: Set up Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@v4
         with:
           go-version: 1
         id: go
@@ -78,7 +78,7 @@ jobs:
           prerelease: false
 
       - name: Upload release assets
-        uses: softprops/action-gh-release@d4e8205d7e959a9107da6396278b2f1f07af0f9b
+        uses: softprops/action-gh-release@c9b46fe7aad9f02afd89b12450b780f52dacfb2d
         if: startsWith(github.ref, 'refs/tags/')
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

dnscrypt-proxy/config.go

@@ -927,7 +927,7 @@ func (config *Config) loadSource(proxy *Proxy, cfgSourceName string, cfgSource *
 		cfgSource.Prefix,
 	)
 	if err != nil {
-		if len(source.in) <= 0 {
+		if len(source.bin) <= 0 {
 			dlog.Criticalf("Unable to retrieve source [%s]: [%s]", cfgSourceName, err)
 			return err
 		}

dnscrypt-proxy/example-dnscrypt-proxy.toml

@@ -256,7 +256,17 @@ cert_refresh_delay = 240
 bootstrap_resolvers = ['9.9.9.11:53', '8.8.8.8:53']
 
 
-## Always use the bootstrap resolver before the system DNS settings.
+## When internal DNS resolution is required, for example to retrieve
+## the resolvers list:
+##
+## - queries will be sent to dnscrypt-proxy itself, if it is already
+##   running with active servers (*)
+## - or else, queries will be sent to fallback servers
+## - finally, if `ignore_system_dns` is `false`, queries will be sent
+##   to the system DNS
+##
+## (*) this is incompatible with systemd sockets.
+## `listen_addrs` must not be empty.
 
 ignore_system_dns = true
 

dnscrypt-proxy/example-forwarding-rules.txt

@@ -23,4 +23,11 @@
 # 192.in-addr.arpa 192.168.1.1
 
 ## Forward queries for example.com and *.example.com to 9.9.9.9 and 8.8.8.8
-# example.com     9.9.9.9,8.8.8.8
+# example.com      9.9.9.9,8.8.8.8
+
+## Forward queries for .onion names to a local Tor client
+## Tor must be configured with the following in the torrc file:
+## DNSPort 9053
+## AutomapHostsOnResolve 1
+
+# onion            127.0.0.1:9053

dnscrypt-proxy/proxy.go

@@ -248,6 +248,8 @@ func (proxy *Proxy) StartProxy() {
 			dlog.Fatal(err)
 		}
 	}
+	proxy.xTransport.internalResolverReady = false
+	proxy.xTransport.internalResolvers = proxy.listenAddresses
 	liveServers, err := proxy.serversInfo.refresh(proxy)
 	if liveServers > 0 {
 		proxy.certIgnoreTimestamp = false

dnscrypt-proxy/serversInfo.go

@@ -242,6 +242,7 @@ func (serversInfo *ServersInfo) refresh(proxy *Proxy) (int, error) {
 	for _, registeredServer := range registeredServers {
 		if err = serversInfo.refreshServer(proxy, registeredServer.name, registeredServer.stamp); err == nil {
 			liveServers++
+			proxy.xTransport.internalResolverReady = true
 		}
 	}
 	serversInfo.Lock()

dnscrypt-proxy/sources.go

@@ -3,10 +3,10 @@ package main
 import (
 	"bytes"
 	"fmt"
+	"math"
 	"math/rand"
 	"net/url"
 	"os"
-	"path/filepath"
 	"strings"
 	"time"
 
@@ -31,13 +31,13 @@ const (
 type Source struct {
 	name                    string
 	urls                    []*url.URL
-	format                  SourceFormat
-	in                      []byte
+	bin                     []byte // copy of the file content - there's something wrong in our logic, we shouldn't need to keep that in memory
 	minisignKey             *minisign.PublicKey
 	cacheFile               string
+	prefix                  string
 	cacheTTL, prefetchDelay time.Duration
 	refresh                 time.Time
-	prefix                  string
+	format                  SourceFormat
 }
 
 func (source *Source) checkSignature(bin, sig []byte) (err error) {
@@ -48,10 +48,10 @@ func (source *Source) checkSignature(bin, sig []byte) (err error) {
 	return err
 }
 
-// timeNow can be replaced by tests to provide a static value
+// timeNow() can be replaced by tests to provide a static value
 var timeNow = time.Now
 
-func (source *Source) fetchFromCache(now time.Time) (delay time.Duration, err error) {
+func (source *Source) fetchFromCache(now time.Time) (remaining time.Duration, err error) {
 	var bin, sig []byte
 	if bin, err = os.ReadFile(source.cacheFile); err != nil {
 		return
@@ -62,14 +62,14 @@ func (source *Source) fetchFromCache(now time.Time) (delay time.Duration, err er
 	if err = source.checkSignature(bin, sig); err != nil {
 		return
 	}
-	source.in = bin
+	source.bin = bin
 	var fi os.FileInfo
 	if fi, err = os.Stat(source.cacheFile); err != nil {
 		return
 	}
 	if elapsed := now.Sub(fi.ModTime()); elapsed < source.cacheTTL {
-		delay = source.prefetchDelay - elapsed
-		dlog.Debugf("Source [%s] cache file [%s] is still fresh, next update: %v", source.name, source.cacheFile, delay)
+		remaining = source.prefetchDelay - elapsed
+		dlog.Debugf("Source [%s] cache file [%s] is still fresh, next update: %v min", source.name, source.cacheFile, math.Round(remaining.Minutes()))
 	} else {
 		dlog.Debugf("Source [%s] cache file [%s] needs to be refreshed", source.name, source.cacheFile)
 	}
@@ -98,25 +98,24 @@ func writeSource(f string, bin, sig []byte) (err error) {
 	return fSig.Commit()
 }
 
-func (source *Source) writeToCache(bin, sig []byte, now time.Time) {
+func (source *Source) updateCache(bin, sig []byte, now time.Time) error {
 	f := source.cacheFile
-	var writeErr error // an error writing cache isn't fatal
-	defer func() {
-		source.in = bin
-		if writeErr == nil {
-			return
-		}
-		if absPath, absErr := filepath.Abs(f); absErr == nil {
-			f = absPath
-		}
-		dlog.Warnf("%s: %s", f, writeErr)
-	}()
-	if !bytes.Equal(source.in, bin) {
-		if writeErr = writeSource(f, bin, sig); writeErr != nil {
-			return
-		}
+	// If the data is unchanged, update the files timestamps only
+	if bytes.Equal(source.bin, bin) {
+		_ = os.Chtimes(f, now, now)
+		_ = os.Chtimes(f+".minisig", now, now)
+		return nil
+	}
+	// Otherwise, write the new data and signature
+	if err := writeSource(f, bin, sig); err != nil {
+		dlog.Warnf("Source [%s] failed to update cache file [%s]: %v", source.name, f, err)
+		return err
 	}
-	writeErr = os.Chtimes(f, now, now)
+	source.bin = bin // In-memory copy of the cache file content
+	// The tests require the timestamps to be updated, no idea why
+	_ = os.Chtimes(f, now, now)
+	_ = os.Chtimes(f+".minisig", now, now)
+	return nil
 }
 
 func (source *Source) parseURLs(urls []string) {
@@ -134,23 +133,23 @@ func fetchFromURL(xTransport *XTransport, u *url.URL) (bin []byte, err error) {
 	return bin, err
 }
 
-func (source *Source) fetchWithCache(xTransport *XTransport, now time.Time) (delay time.Duration, err error) {
-	if delay, err = source.fetchFromCache(now); err != nil {
+func (source *Source) fetchWithCache(xTransport *XTransport, now time.Time) (time.Duration, error) {
+	remaining, err := source.fetchFromCache(now)
+	if err != nil {
 		if len(source.urls) == 0 {
-			dlog.Errorf("Source [%s] cache file [%s] not present and no valid URL", source.name, source.cacheFile)
-			return
+			dlog.Fatalf("Source [%s] cache file [%s] not present and no valid URL", source.name, source.cacheFile)
+			return 0, err
 		}
 		dlog.Debugf("Source [%s] cache file [%s] not present", source.name, source.cacheFile)
 	}
-	if len(source.urls) > 0 {
-		defer func() {
-			source.refresh = now.Add(delay)
-		}()
+	if len(source.urls) == 0 {
+		dlog.Debugf("No URL to update [%s]", source.name)
+		return 24 * time.Hour, nil
 	}
-	if len(source.urls) == 0 || delay > 0 {
-		return
+	if remaining > 0 {
+		source.refresh = now.Add(remaining)
+		return remaining, nil
 	}
-	delay = MinimumPrefetchInterval
 	var bin, sig []byte
 	for _, srcURL := range source.urls {
 		dlog.Infof("Source [%s] loading from URL [%s]", source.name, srcURL)
@@ -171,11 +170,13 @@ func (source *Source) fetchWithCache(xTransport *XTransport, now time.Time) (del
 		dlog.Debugf("Source [%s] failed signature check using URL [%s]", source.name, srcURL)
 	}
 	if err != nil {
-		return
+		source.refresh = now.Add(MinimumPrefetchInterval)
+		return MinimumPrefetchInterval, err
 	}
-	source.writeToCache(bin, sig, now)
-	delay = source.prefetchDelay
-	return
+	source.updateCache(bin, sig, now)
+	remaining = source.prefetchDelay
+	source.refresh = now.Add(remaining)
+	return remaining, nil
 }
 
 // NewSource loads a new source using the given cacheFile and urls, ensuring it has a valid signature
@@ -229,7 +230,7 @@ func PrefetchSources(xTransport *XTransport, sources []*Source) time.Duration {
 		if delay, err := source.fetchWithCache(xTransport, now); err != nil {
 			dlog.Infof("Prefetching [%s] failed: %v, will retry in %v", source.name, err, interval)
 		} else {
-			dlog.Debugf("Prefetching [%s] succeeded, next update: %v", source.name, delay)
+			dlog.Debugf("Prefetching [%s] succeeded, next update: %v min", source.name, math.Round(delay.Minutes()))
 			if delay >= MinimumPrefetchInterval && (interval == MinimumPrefetchInterval || interval > delay) {
 				interval = delay
 			}
@@ -254,8 +255,8 @@ func (source *Source) parseV2() ([]RegisteredServer, error) {
 		stampErrs = append(stampErrs, stampErr)
 		dlog.Warn(stampErr)
 	}
-	in := string(source.in)
-	parts := strings.Split(in, "## ")
+	bin := string(source.bin)
+	parts := strings.Split(bin, "## ")
 	if len(parts) < 2 {
 		return registeredServers, fmt.Errorf("Invalid format for source at [%v]", source.urls)
 	}

dnscrypt-proxy/sources_test.go

@@ -284,9 +284,9 @@ func prepSourceTestCache(t *testing.T, d *SourceTestData, e *SourceTestExpect, s
 	e.cache = []SourceFixture{d.fixtures[state][source], d.fixtures[state][source+".minisig"]}
 	switch state {
 	case TestStateCorrect:
-		e.Source.in, e.success = e.cache[0].content, true
+		e.Source.bin, e.success = e.cache[0].content, true
 	case TestStateExpired:
-		e.Source.in = e.cache[0].content
+		e.Source.bin = e.cache[0].content
 	case TestStatePartial, TestStatePartialSig:
 		e.err = "signature"
 	case TestStateMissing, TestStateMissingSig, TestStateOpenErr, TestStateOpenSigErr:
@@ -339,7 +339,7 @@ func prepSourceTestDownload(
 		switch state {
 		case TestStateCorrect:
 			e.cache = []SourceFixture{d.fixtures[state][source], d.fixtures[state][source+".minisig"]}
-			e.Source.in, e.success = e.cache[0].content, true
+			e.Source.bin, e.success = e.cache[0].content, true
 			fallthrough
 		case TestStateMissingSig, TestStatePartial, TestStatePartialSig, TestStateReadSigErr:
 			d.reqExpect[path+".minisig"]++
@@ -477,7 +477,7 @@ func TestPrefetchSources(t *testing.T) {
 			e.mtime = d.timeUpd
 			s := &Source{}
 			*s = *e.Source
-			s.in = nil
+			s.bin = nil
 			sources = append(sources, s)
 			expects = append(expects, e)
 		}