diff --git commands/pm/pm.drush.inc commands/pm/pm.drush.inc
index 2420e43..6f80da6 100644
--- commands/pm/pm.drush.inc
+++ commands/pm/pm.drush.inc
@@ -1369,17 +1369,24 @@ function pm_dl_destination($type) {
     if (empty($destination) && $site_root && $in_site_directory) {
       $destination = pm_dl_destination_lookup($type, $drupal_root, $full_site_root);
     }
-    // Attempt 2: If the destination directory already exists for sites/all,
+    // Attempt 2: If a specific (non default) site directory exists, then:
+    // - if sites/specific_site/modules|themes directory also exists we use that
+    // - if sites/all/modules|themes directory does not exist or exist but
+    //   contains only one file (potentially README.txt) we create & use
+    //   destination in the site specific directory.
+    if (empty($destination) && $site_root && $site_root !== 'sites/default' && is_dir($full_site_root)) {
+      $make_site_specific_dest = FALSE;
+      if ($type == 'module' || $type == 'theme') {
+        $sites_all_type = $sites_all . '/' . $type . 's';
+        $make_site_specific_dest = is_dir($sites_all_type) ? (count(scandir($sites_all_type)) < 4) : TRUE;
+      }
+      $destination = pm_dl_destination_lookup($type, $drupal_root, $full_site_root, $make_site_specific_dest);
+    }
+    // Attempt 3: If the destination directory already exists for sites/all,
     // then we use that.
     if (empty($destination) && $drupal_root) {
       $destination = pm_dl_destination_lookup($type, $drupal_root, $sites_all);
     }
-    // Attempt 3: If a specific (non default) site directory exists and
-    // sites/all does not exist, then we create destination in the site
-    // specific directory.
-    if (empty($destination) && $site_root && $site_root !== 'sites/default' && is_dir($full_site_root) && !is_dir($sites_all)) {
-      $destination = pm_dl_destination_lookup($type, $drupal_root, $full_site_root, TRUE);
-    }
     // Attempt 4: If sites/all exists, then we create destination in the
     // sites/all directory.
     if (empty($destination) && is_dir($sites_all)) {