diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index c9e75eab4..cc771ee89 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -113,6 +113,12 @@ def parseOpts(overrideArguments=None):
                 pass
         return opts
 
+    xdg_cache_home = os.environ.get('XDG_CACHE_HOME')
+    if xdg_cache_home:
+        userCacheDir = os.path.join(xdg_cache_home, 'youtube-dl')
+    else:
+        userCacheDir = os.path.join(os.path.expanduser('~'), '.cache', 'youtube-dl')
+
     max_width = 80
     max_help_position = 80
 
@@ -168,7 +174,7 @@ def parseOpts(overrideArguments=None):
     general.add_option('--proxy', dest='proxy', default=None, help='Use the specified HTTP/HTTPS proxy', metavar='URL')
     general.add_option('--no-check-certificate', action='store_true', dest='no_check_certificate', default=False, help='Suppress HTTPS certificate validation.')
     general.add_option(
-        '--cache-dir', dest='cachedir', default=u'~/.youtube-dl/cache',
+        '--cache-dir', dest='cachedir', default=userCacheDir,
         help='Location in the filesystem where youtube-dl can store downloaded information permanently. %default by default')
     general.add_option(
         '--no-cache-dir', action='store_const', const=None, dest='cachedir',
@@ -370,9 +376,13 @@ def parseOpts(overrideArguments=None):
     else:
         xdg_config_home = os.environ.get('XDG_CONFIG_HOME')
         if xdg_config_home:
-            userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
+            userConfFile = os.path.join(xdg_config_home, 'youtube-dl', 'config')
+            if not os.path.isfile(userConfFile):
+                userConfFile = os.path.join(xdg_config_home, 'youtube-dl.conf')
         else:
-            userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
+            userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl', 'config')
+            if not os.path.isfile(userConfFile):
+                userConfFile = os.path.join(os.path.expanduser('~'), '.config', 'youtube-dl.conf')
         systemConf = _readOptions('/etc/youtube-dl.conf')
         userConf = _readOptions(userConfFile)
         commandLineConf = sys.argv[1:]
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 4389924c5..f3d279210 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -421,8 +421,12 @@ class YoutubeIE(YoutubeBaseInfoExtractor, SubtitlesInfoExtractor):
         # Read from filesystem cache
         func_id = '%s_%s_%d' % (player_type, player_id, slen)
         assert os.path.basename(func_id) == func_id
-        cache_dir = self._downloader.params.get('cachedir',
-                                                u'~/.youtube-dl/cache')
+        xdg_cache_home = os.environ.get('XDG_CACHE_HOME')
+        if xdg_cache_home:
+            userCacheDir = os.path.join(xdg_cache_home, 'youtube-dl')
+        else:
+            userCacheDir = os.path.join(os.path.expanduser('~'), '.cache', 'youtube-dl')
+        cache_dir = self._downloader.params.get('cachedir', userCacheDir)
 
         cache_enabled = cache_dir is not None
         if cache_enabled: