diff --git a/ext/fg/js/document.js b/ext/fg/js/document.js
index 60b1b9bd..079a5034 100644
--- a/ext/fg/js/document.js
+++ b/ext/fg/js/document.js
@@ -89,8 +89,18 @@ function docImposterCreate(element, isTextarea) {
     return [imposter, container];
 }
 
-function docRangeFromPoint({x, y}, options) {
-    const elements = document.elementsFromPoint(x, y);
+function docElementsFromPoint(x, y, all) {
+    if (all) {
+        return document.elementsFromPoint(x, y);
+    }
+
+    const e = document.elementFromPoint(x, y);
+    return e !== null ? [e] : [];
+}
+
+function docRangeFromPoint(x, y, options) {
+    const deepDomScan = options.scanning.deepDomScan;
+    const elements = docElementsFromPoint(x, y, deepDomScan);
     let imposter = null;
     let imposterContainer = null;
     if (elements.length > 0) {
@@ -108,7 +118,7 @@ function docRangeFromPoint({x, y}, options) {
         }
     }
 
-    const range = caretRangeFromPointExt(x, y, options.scanning.deepDomScan ? elements : []);
+    const range = caretRangeFromPointExt(x, y, deepDomScan ? elements : []);
     if (range !== null) {
         if (imposter !== null) {
             docSetImposterStyle(imposterContainer.style, 'z-index', '-2147483646');
diff --git a/ext/fg/js/frontend.js b/ext/fg/js/frontend.js
index 0b60aa2b..c98a9a33 100644
--- a/ext/fg/js/frontend.js
+++ b/ext/fg/js/frontend.js
@@ -21,8 +21,6 @@ class Frontend {
     constructor(popup, ignoreNodes) {
         this.popup = popup;
         this.popupTimer = null;
-        this.mouseDownLeft = false;
-        this.mouseDownMiddle = false;
         this.textSourceLast = null;
         this.pendingLookup = false;
         this.options = null;
@@ -61,7 +59,6 @@ class Frontend {
             window.addEventListener('mousemove', this.onMouseMove.bind(this));
             window.addEventListener('mouseover', this.onMouseOver.bind(this));
             window.addEventListener('mouseout', this.onMouseOut.bind(this));
-            window.addEventListener('mouseup', this.onMouseUp.bind(this));
             window.addEventListener('resize', this.onResize.bind(this));
 
             if (this.options.scanning.touchInputEnabled) {
@@ -80,7 +77,7 @@ class Frontend {
     }
 
     onMouseOver(e) {
-        if (e.target === this.popup.container && this.popupTimer) {
+        if (e.target === this.popup.container && this.popupTimer !== null) {
             this.popupTimerClear();
         }
     }
@@ -88,38 +85,32 @@ class Frontend {
     onMouseMove(e) {
         this.popupTimerClear();
 
-        if (!this.options.general.enable) {
+        if (
+            this.pendingLookup ||
+            !this.options.general.enable ||
+            (e.buttons & 0x1) !== 0x0 // Left mouse button
+        ) {
             return;
         }
 
-        if (this.mouseDownLeft) {
-            return;
-        }
-
-        if (this.pendingLookup) {
-            return;
-        }
-
-        const mouseScan = this.mouseDownMiddle && this.options.scanning.middleMouse;
-        const keyScan =
-            this.options.scanning.modifier === 'alt' && e.altKey ||
-            this.options.scanning.modifier === 'ctrl' && e.ctrlKey ||
-            this.options.scanning.modifier === 'shift' && e.shiftKey ||
-            this.options.scanning.modifier === 'none';
-
-        if (!keyScan && !mouseScan) {
+        const scanningOptions = this.options.scanning;
+        const scanningModifier = scanningOptions.modifier;
+        if (!(
+            Frontend.isScanningModifierPressed(scanningModifier, e) ||
+            (scanningOptions.middleMouse && (e.buttons & 0x4) !== 0x0) // Middle mouse button
+        )) {
             return;
         }
 
         const search = async () => {
             try {
-                await this.searchAt({x: e.clientX, y: e.clientY}, 'mouse');
+                await this.searchAt(e.clientX, e.clientY, 'mouse');
             } catch (e) {
                 this.onError(e);
             }
         };
 
-        if (this.options.scanning.modifier === 'none') {
+        if (scanningModifier === 'none') {
             this.popupTimerSet(search);
         } else {
             search();
@@ -135,23 +126,8 @@ class Frontend {
             return false;
         }
 
-        this.mousePosLast = {x: e.clientX, y: e.clientY};
         this.popupTimerClear();
         this.searchClear();
-
-        if (e.which === 1) {
-            this.mouseDownLeft = true;
-        } else if (e.which === 2) {
-            this.mouseDownMiddle = true;
-        }
-    }
-
-    onMouseUp(e) {
-        if (e.which === 1) {
-            this.mouseDownLeft = false;
-        } else if (e.which === 2) {
-            this.mouseDownMiddle = false;
-        }
     }
 
     onMouseOut(e) {
@@ -243,8 +219,8 @@ class Frontend {
         }
     }
 
-    onAfterSearch(newRange, type, searched, success) {
-        if (type === 'mouse') {
+    onAfterSearch(newRange, cause, searched, success) {
+        if (cause === 'mouse') {
             return;
         }
 
@@ -254,7 +230,7 @@ class Frontend {
             return;
         }
 
-        if (type === 'touchStart' && newRange !== null) {
+        if (cause === 'touchStart' && newRange !== null) {
             this.scrollPrevent = true;
         }
 
@@ -293,23 +269,22 @@ class Frontend {
     }
 
     popupTimerSet(callback) {
-        this.popupTimerClear();
         this.popupTimer = window.setTimeout(callback, this.options.scanning.delay);
     }
 
     popupTimerClear() {
-        if (this.popupTimer) {
+        if (this.popupTimer !== null) {
             window.clearTimeout(this.popupTimer);
             this.popupTimer = null;
         }
     }
 
-    async searchAt(point, type) {
-        if (this.pendingLookup || await this.popup.containsPoint(point)) {
+    async searchAt(x, y, cause) {
+        if (this.pendingLookup || await this.popup.containsPoint(x, y)) {
             return;
         }
 
-        const textSource = docRangeFromPoint(point, this.options);
+        const textSource = docRangeFromPoint(x, y, this.options);
         let hideResults = textSource === null;
         let searched = false;
         let success = false;
@@ -318,7 +293,7 @@ class Frontend {
             if (!hideResults && (!this.textSourceLast || !this.textSourceLast.equals(textSource))) {
                 searched = true;
                 this.pendingLookup = true;
-                const focus = (type === 'mouse');
+                const focus = (cause === 'mouse');
                 hideResults = !await this.searchTerms(textSource, focus) && !await this.searchKanji(textSource, focus);
                 success = true;
             }
@@ -343,7 +318,7 @@ class Frontend {
             }
 
             this.pendingLookup = false;
-            this.onAfterSearch(this.textSourceLast, type, searched, success);
+            this.onAfterSearch(this.textSourceLast, cause, searched, success);
         }
     }
 
@@ -485,7 +460,7 @@ class Frontend {
         this.clickPrevent = value;
     }
 
-    searchFromTouch(x, y, type) {
+    searchFromTouch(x, y, cause) {
         this.popupTimerClear();
 
         if (!this.options.general.enable || this.pendingLookup) {
@@ -494,7 +469,7 @@ class Frontend {
 
         const search = async () => {
             try {
-                await this.searchAt({x, y}, type);
+                await this.searchAt(x, y, cause);
             } catch (e) {
                 this.onError(e);
             }
@@ -531,6 +506,16 @@ class Frontend {
             textSource.setEndOffset(length);
         }
     }
+
+    static isScanningModifierPressed(scanningModifier, mouseEvent) {
+        switch (scanningModifier) {
+            case 'alt': return mouseEvent.altKey;
+            case 'ctrl': return mouseEvent.ctrlKey;
+            case 'shift': return mouseEvent.shiftKey;
+            case 'none': return true;
+            default: return false;
+        }
+    }
 }
 
 window.yomichan_frontend = Frontend.create();
diff --git a/ext/fg/js/popup-proxy-host.js b/ext/fg/js/popup-proxy-host.js
index 1f14a06f..396f7556 100644
--- a/ext/fg/js/popup-proxy-host.js
+++ b/ext/fg/js/popup-proxy-host.js
@@ -42,7 +42,7 @@ class PopupProxyHost {
             showOrphaned: ({id, elementRect, options}) => this.show(id, elementRect, options),
             hide: ({id}) => this.hide(id),
             setVisible: ({id, visible}) => this.setVisible(id, visible),
-            containsPoint: ({id, point}) => this.containsPoint(id, point),
+            containsPoint: ({id, x, y}) => this.containsPoint(id, x, y),
             termsShow: ({id, elementRect, writingMode, definitions, options, context}) => this.termsShow(id, elementRect, writingMode, definitions, options, context),
             kanjiShow: ({id, elementRect, writingMode, definitions, options, context}) => this.kanjiShow(id, elementRect, writingMode, definitions, options, context),
             clearAutoPlayTimer: ({id}) => this.clearAutoPlayTimer(id)
@@ -108,9 +108,9 @@ class PopupProxyHost {
         return popup.setVisible(visible);
     }
 
-    async containsPoint(id, point) {
+    async containsPoint(id, x, y) {
         const popup = this.getPopup(id);
-        return await popup.containsPoint(point);
+        return await popup.containsPoint(x, y);
     }
 
     async termsShow(id, elementRect, writingMode, definitions, options, context) {
diff --git a/ext/fg/js/popup-proxy.js b/ext/fg/js/popup-proxy.js
index c3a7bff0..f04e24e0 100644
--- a/ext/fg/js/popup-proxy.js
+++ b/ext/fg/js/popup-proxy.js
@@ -69,11 +69,11 @@ class PopupProxy {
         return await this.invokeHostApi('setVisible', {id, visible});
     }
 
-    async containsPoint(point) {
+    async containsPoint(x, y) {
         if (this.id === null) {
             return false;
         }
-        return await this.invokeHostApi('containsPoint', {id: this.id, point});
+        return await this.invokeHostApi('containsPoint', {id: this.id, x, y});
     }
 
     async termsShow(elementRect, writingMode, definitions, options, context) {
diff --git a/ext/fg/js/popup.js b/ext/fg/js/popup.js
index 08c4bfcb..8953cf30 100644
--- a/ext/fg/js/popup.js
+++ b/ext/fg/js/popup.js
@@ -254,7 +254,7 @@ class Popup {
         }
     }
 
-    async containsPoint({x, y}) {
+    async containsPoint(x, y) {
         for (let popup = this; popup !== null && popup.isVisible(); popup = popup.child) {
             const rect = popup.container.getBoundingClientRect();
             if (x >= rect.left && y >= rect.top && x < rect.right && y < rect.bottom) {
diff --git a/ext/mixed/js/display.js b/ext/mixed/js/display.js
index eca67b5e..ca1738a6 100644
--- a/ext/mixed/js/display.js
+++ b/ext/mixed/js/display.js
@@ -81,7 +81,7 @@ class Display {
             const {docRangeFromPoint, docSentenceExtract} = this.dependencies;
 
             const clickedElement = $(e.target);
-            const textSource = docRangeFromPoint({x: e.clientX, y: e.clientY}, this.options);
+            const textSource = docRangeFromPoint(e.clientX, e.clientY, this.options);
             if (textSource === null) {
                 return false;
             }